2024-11-20 17:46:58,236 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 17:46:58,265 main DEBUG Took 0.026647 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 17:46:58,266 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 17:46:58,266 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 17:46:58,267 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 17:46:58,269 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,283 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 17:46:58,300 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,303 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,304 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,304 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,306 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,306 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,307 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,309 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,310 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,311 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,313 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,314 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,314 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,315 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,316 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:46:58,317 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,317 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 17:46:58,319 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:46:58,322 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 17:46:58,324 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 17:46:58,325 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 17:46:58,327 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 17:46:58,327 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 17:46:58,337 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 17:46:58,341 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 17:46:58,344 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 17:46:58,344 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 17:46:58,345 main DEBUG createAppenders(={Console}) 2024-11-20 17:46:58,346 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-20 17:46:58,346 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 17:46:58,347 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-20 17:46:58,348 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 17:46:58,348 main DEBUG OutputStream closed 2024-11-20 17:46:58,349 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 17:46:58,349 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 17:46:58,350 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-20 17:46:58,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 17:46:58,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 17:46:58,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 17:46:58,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 17:46:58,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 17:46:58,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 17:46:58,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 17:46:58,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 17:46:58,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 17:46:58,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 17:46:58,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 17:46:58,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 17:46:58,460 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 17:46:58,460 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 17:46:58,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 17:46:58,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 17:46:58,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 17:46:58,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 17:46:58,465 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 17:46:58,465 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-20 17:46:58,466 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 17:46:58,467 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-20T17:46:58,481 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-20 17:46:58,484 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 17:46:58,485 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T17:46:58,726 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05 2024-11-20T17:46:58,752 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=10129 2024-11-20T17:46:58,864 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T17:46:58,996 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:46:59,025 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/cluster_538bea00-2849-a622-e71b-ceb665affad4, deleteOnExit=true 2024-11-20T17:46:59,149 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/cluster_538bea00-2849-a622-e71b-ceb665affad4/zookeeper_0, clientPort=59586, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/cluster_538bea00-2849-a622-e71b-ceb665affad4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/cluster_538bea00-2849-a622-e71b-ceb665affad4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T17:46:59,162 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59586 2024-11-20T17:46:59,163 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:46:59,256 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec with version=8 2024-11-20T17:46:59,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/hbase-staging 2024-11-20T17:46:59,345 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T17:46:59,678 INFO [Time-limited test {}] client.ConnectionUtils(128): master/613581fcdc40:0 server-side Connection retries=45 2024-11-20T17:46:59,688 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:46:59,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:46:59,699 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:46:59,699 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:46:59,700 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:46:59,848 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T17:46:59,953 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T17:47:00,013 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T17:47:00,027 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T17:47:00,032 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:47:00,074 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15948 (auto-detected) 2024-11-20T17:47:00,075 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T17:47:00,105 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37951 2024-11-20T17:47:00,138 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37951 connecting to ZooKeeper ensemble=127.0.0.1:59586 2024-11-20T17:47:00,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379510x0, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:47:00,331 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37951-0x101595b3a5c0000 connected 2024-11-20T17:47:00,862 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:00,864 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:00,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:00,876 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec, hbase.cluster.distributed=false 2024-11-20T17:47:00,902 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:47:00,907 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37951 2024-11-20T17:47:00,907 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37951 2024-11-20T17:47:00,907 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37951 2024-11-20T17:47:00,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37951 2024-11-20T17:47:00,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37951 2024-11-20T17:47:00,983 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/613581fcdc40:0 server-side Connection retries=45 2024-11-20T17:47:00,984 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:00,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:00,985 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:47:00,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:00,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:47:00,988 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:47:00,990 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:47:00,991 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33139 2024-11-20T17:47:00,993 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33139 connecting to ZooKeeper ensemble=127.0.0.1:59586 2024-11-20T17:47:00,994 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:00,995 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:01,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:331390x0, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:47:01,090 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33139-0x101595b3a5c0001 connected 2024-11-20T17:47:01,090 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:331390x0, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:01,099 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:47:01,113 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.cache 2024-11-20T17:47:01,118 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T17:47:01,119 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T17:47:01,121 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T17:47:01,122 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T17:47:01,122 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T17:47:01,127 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T17:47:01,131 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T17:47:01,131 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.persistence is complete. 2024-11-20T17:47:01,133 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:47:01,136 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T17:47:01,143 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:47:01,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33139 2024-11-20T17:47:01,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33139 2024-11-20T17:47:01,149 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33139 2024-11-20T17:47:01,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33139 2024-11-20T17:47:01,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33139 2024-11-20T17:47:01,176 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/613581fcdc40:0 server-side Connection retries=45 2024-11-20T17:47:01,177 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:01,177 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:01,177 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:47:01,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:01,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:47:01,178 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:47:01,179 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:47:01,182 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36667 2024-11-20T17:47:01,185 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36667 connecting to ZooKeeper ensemble=127.0.0.1:59586 2024-11-20T17:47:01,185 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:01,186 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:01,249 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366670x0, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:47:01,251 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:366670x0, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:01,251 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:47:01,263 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36667-0x101595b3a5c0002 connected 2024-11-20T17:47:01,264 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.cache 2024-11-20T17:47:01,264 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T17:47:01,265 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T17:47:01,265 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T17:47:01,268 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T17:47:01,269 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T17:47:01,269 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T17:47:01,270 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.persistence is complete. 2024-11-20T17:47:01,270 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T17:47:01,273 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:47:01,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T17:47:01,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:47:01,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36667 2024-11-20T17:47:01,288 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36667 2024-11-20T17:47:01,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36667 2024-11-20T17:47:01,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36667 2024-11-20T17:47:01,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36667 2024-11-20T17:47:01,336 DEBUG [M:0;613581fcdc40:37951 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;613581fcdc40:37951 2024-11-20T17:47:01,338 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/613581fcdc40,37951,1732124819468 2024-11-20T17:47:01,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:01,360 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:01,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:01,366 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/613581fcdc40,37951,1732124819468 2024-11-20T17:47:01,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:01,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T17:47:01,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:01,402 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T17:47:01,402 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:01,403 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T17:47:01,405 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/613581fcdc40,37951,1732124819468 from backup master directory 2024-11-20T17:47:01,411 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:01,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:01,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/613581fcdc40,37951,1732124819468 2024-11-20T17:47:01,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:01,413 WARN [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:47:01,414 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=613581fcdc40,37951,1732124819468 2024-11-20T17:47:01,417 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T17:47:01,419 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T17:47:01,431 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/hbase.id] with ID: 2282afdf-e043-4617-892c-c46cc1d3c615 2024-11-20T17:47:01,432 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/.tmp/hbase.id 2024-11-20T17:47:01,439 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/hbase.id] 2024-11-20T17:47:01,484 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:01,491 INFO [master/613581fcdc40:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T17:47:01,519 INFO [master/613581fcdc40:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-20T17:47:01,527 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:01,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:01,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:01,563 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:47:01,568 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T17:47:01,622 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:01,633 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:01,713 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store 2024-11-20T17:47:01,742 INFO [master/613581fcdc40:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T17:47:01,746 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:01,749 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T17:47:01,749 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:01,749 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:01,751 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T17:47:01,752 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:01,752 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:01,753 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732124821748Disabling compacts and flushes for region at 1732124821748Disabling writes for close at 1732124821752 (+4 ms)Writing region close event to WAL at 1732124821752Closed at 1732124821752 2024-11-20T17:47:01,754 WARN [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/.initializing 2024-11-20T17:47:01,754 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/WALs/613581fcdc40,37951,1732124819468 2024-11-20T17:47:01,774 INFO [master/613581fcdc40:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:47:01,791 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=613581fcdc40%2C37951%2C1732124819468, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/WALs/613581fcdc40,37951,1732124819468, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/oldWALs, maxLogs=10 2024-11-20T17:47:01,855 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/WALs/613581fcdc40,37951,1732124819468/613581fcdc40%2C37951%2C1732124819468.1732124821798 2024-11-20T17:47:01,856 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:01,857 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:01,858 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:01,863 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:01,865 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:01,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:01,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T17:47:01,970 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:01,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:01,977 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:01,994 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T17:47:01,994 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:01,995 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:01,995 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:02,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T17:47:02,030 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:02,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:02,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:02,041 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T17:47:02,041 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:02,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:02,044 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:02,047 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:02,048 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:02,051 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:02,052 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:02,055 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:47:02,057 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:02,066 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:47:02,067 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74659855, jitterRate=0.11251853406429291}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:47:02,076 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732124821885Initializing all the Stores at 1732124821886 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124821886Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124821889 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124821890 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124821893 (+3 ms)Cleaning up temporary data from old regions at 1732124822052 (+159 ms)Region opened successfully at 1732124822076 (+24 ms) 2024-11-20T17:47:02,078 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T17:47:02,127 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f2e74c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:47:02,178 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T17:47:02,196 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T17:47:02,196 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T17:47:02,201 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:47:02,203 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T17:47:02,213 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T17:47:02,213 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T17:47:02,251 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T17:47:02,268 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T17:47:02,316 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T17:47:02,319 INFO [master/613581fcdc40:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T17:47:02,322 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T17:47:02,327 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T17:47:02,330 INFO [master/613581fcdc40:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T17:47:02,335 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T17:47:02,348 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T17:47:02,350 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T17:47:02,358 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T17:47:02,378 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T17:47:02,390 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T17:47:02,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:02,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:02,412 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:02,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:02,412 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:02,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:02,415 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=613581fcdc40,37951,1732124819468, sessionid=0x101595b3a5c0000, setting cluster-up flag (Was=false) 2024-11-20T17:47:02,443 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:02,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:02,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:02,474 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T17:47:02,476 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=613581fcdc40,37951,1732124819468 2024-11-20T17:47:02,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:02,495 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:02,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:02,527 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T17:47:02,530 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=613581fcdc40,37951,1732124819468 2024-11-20T17:47:02,535 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T17:47:02,618 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(746): ClusterId : 2282afdf-e043-4617-892c-c46cc1d3c615 2024-11-20T17:47:02,618 INFO [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(746): ClusterId : 2282afdf-e043-4617-892c-c46cc1d3c615 2024-11-20T17:47:02,621 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:47:02,621 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:47:02,631 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:02,644 INFO [master/613581fcdc40:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T17:47:02,645 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:47:02,645 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:47:02,645 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:47:02,645 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:47:02,651 INFO [master/613581fcdc40:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T17:47:02,658 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 613581fcdc40,37951,1732124819468 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T17:47:02,665 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:47:02,665 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:47:02,666 DEBUG [RS:1;613581fcdc40:36667 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b226e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:47:02,666 DEBUG [RS:0;613581fcdc40:33139 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4410a8cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:47:02,672 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:02,673 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:02,673 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:02,673 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:02,674 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/613581fcdc40:0, corePoolSize=10, maxPoolSize=10 2024-11-20T17:47:02,674 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,674 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:47:02,675 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,684 DEBUG [RS:1;613581fcdc40:36667 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;613581fcdc40:36667 2024-11-20T17:47:02,685 DEBUG [RS:0;613581fcdc40:33139 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;613581fcdc40:33139 2024-11-20T17:47:02,689 INFO [RS:1;613581fcdc40:36667 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:47:02,689 INFO [RS:0;613581fcdc40:33139 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:47:02,690 INFO [RS:1;613581fcdc40:36667 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:47:02,690 INFO [RS:0;613581fcdc40:33139 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:47:02,690 DEBUG [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:47:02,690 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732124852690 2024-11-20T17:47:02,692 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T17:47:02,693 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T17:47:02,694 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(2659): reportForDuty to master=613581fcdc40,37951,1732124819468 with port=36667, startcode=1732124821176 2024-11-20T17:47:02,692 DEBUG [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:47:02,697 INFO [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(2659): reportForDuty to master=613581fcdc40,37951,1732124819468 with port=33139, startcode=1732124820951 2024-11-20T17:47:02,699 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T17:47:02,700 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T17:47:02,700 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T17:47:02,700 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T17:47:02,701 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:02,701 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T17:47:02,701 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,707 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:02,708 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T17:47:02,707 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T17:47:02,709 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T17:47:02,710 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T17:47:02,713 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T17:47:02,714 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T17:47:02,715 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732124822715,5,FailOnTimeoutGroup] 2024-11-20T17:47:02,716 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732124822716,5,FailOnTimeoutGroup] 2024-11-20T17:47:02,716 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,716 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T17:47:02,717 DEBUG [RS:1;613581fcdc40:36667 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:47:02,717 DEBUG [RS:0;613581fcdc40:33139 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:47:02,718 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,718 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,735 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T17:47:02,735 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec 2024-11-20T17:47:02,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:02,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T17:47:02,756 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:47:02,756 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:02,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:02,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T17:47:02,760 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:47:02,760 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:02,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:02,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T17:47:02,772 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:47:02,772 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:02,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:02,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T17:47:02,778 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:47:02,778 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:02,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:02,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:47:02,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:02,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:02,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:47:02,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:47:02,781 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:47:02,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:47:02,789 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:47:02,789 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66840705, jitterRate=-0.003995880484580994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:47:02,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732124822750Initializing all the Stores at 1732124822750Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124822750Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124822751 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124822751Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124822751Cleaning up temporary data from old regions at 1732124822781 (+30 ms)Region opened successfully at 1732124822794 (+13 ms) 2024-11-20T17:47:02,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:47:02,795 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:47:02,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:47:02,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:47:02,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:47:02,797 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51645, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:47:02,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37749, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:47:02,797 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:47:02,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732124822795Disabling compacts and flushes for region at 1732124822795Disabling writes for close at 1732124822795Writing region close event to WAL at 1732124822797 (+2 ms)Closed at 1732124822797 2024-11-20T17:47:02,800 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:02,800 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T17:47:02,805 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37951 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 613581fcdc40,36667,1732124821176 2024-11-20T17:47:02,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T17:47:02,807 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37951 {}] master.ServerManager(517): Registering regionserver=613581fcdc40,36667,1732124821176 2024-11-20T17:47:02,815 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:47:02,818 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T17:47:02,818 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37951 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 613581fcdc40,33139,1732124820951 2024-11-20T17:47:02,818 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37951 {}] master.ServerManager(517): Registering regionserver=613581fcdc40,33139,1732124820951 2024-11-20T17:47:02,822 DEBUG [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec 2024-11-20T17:47:02,822 DEBUG [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec 2024-11-20T17:47:02,822 DEBUG [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T17:47:02,822 DEBUG [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T17:47:02,822 DEBUG [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:47:02,822 DEBUG [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:47:02,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:47:02,907 DEBUG [RS:0;613581fcdc40:33139 {}] zookeeper.ZKUtil(111): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/613581fcdc40,33139,1732124820951 2024-11-20T17:47:02,907 DEBUG [RS:1;613581fcdc40:36667 {}] zookeeper.ZKUtil(111): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/613581fcdc40,36667,1732124821176 2024-11-20T17:47:02,907 WARN [RS:0;613581fcdc40:33139 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:47:02,907 WARN [RS:1;613581fcdc40:36667 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:47:02,907 INFO [RS:1;613581fcdc40:36667 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:02,907 INFO [RS:0;613581fcdc40:33139 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:02,908 DEBUG [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,36667,1732124821176 2024-11-20T17:47:02,908 DEBUG [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,33139,1732124820951 2024-11-20T17:47:02,910 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [613581fcdc40,36667,1732124821176] 2024-11-20T17:47:02,910 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [613581fcdc40,33139,1732124820951] 2024-11-20T17:47:02,933 INFO [RS:0;613581fcdc40:33139 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:47:02,933 INFO [RS:1;613581fcdc40:36667 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:47:02,950 INFO [RS:0;613581fcdc40:33139 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:47:02,950 INFO [RS:1;613581fcdc40:36667 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:47:02,955 INFO [RS:1;613581fcdc40:36667 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:47:02,955 INFO [RS:0;613581fcdc40:33139 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:47:02,956 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,956 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,957 INFO [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:47:02,957 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:47:02,962 INFO [RS:0;613581fcdc40:33139 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:47:02,962 INFO [RS:1;613581fcdc40:36667 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:47:02,963 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,963 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,964 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:47:02,964 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,964 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:47:02,965 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,965 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,965 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,965 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,965 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,965 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,965 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,965 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,965 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,965 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:02,965 DEBUG [RS:0;613581fcdc40:33139 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:02,965 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,966 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:02,966 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:02,966 DEBUG [RS:1;613581fcdc40:36667 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:02,968 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,968 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,969 WARN [613581fcdc40:37951 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:47:02,969 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,969 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,969 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,969 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,969 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,969 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,969 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,36667,1732124821176-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:47:02,969 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,969 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:02,970 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,33139,1732124820951-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:47:02,998 INFO [RS:0;613581fcdc40:33139 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:47:02,998 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:47:03,000 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,36667,1732124821176-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,000 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,33139,1732124820951-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,001 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,001 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,001 INFO [RS:1;613581fcdc40:36667 {}] regionserver.Replication(171): 613581fcdc40,36667,1732124821176 started 2024-11-20T17:47:03,001 INFO [RS:0;613581fcdc40:33139 {}] regionserver.Replication(171): 613581fcdc40,33139,1732124820951 started 2024-11-20T17:47:03,029 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,029 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,030 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(1482): Serving as 613581fcdc40,36667,1732124821176, RpcServer on 613581fcdc40/172.17.0.2:36667, sessionid=0x101595b3a5c0002 2024-11-20T17:47:03,030 INFO [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(1482): Serving as 613581fcdc40,33139,1732124820951, RpcServer on 613581fcdc40/172.17.0.2:33139, sessionid=0x101595b3a5c0001 2024-11-20T17:47:03,031 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:47:03,031 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:47:03,031 DEBUG [RS:1;613581fcdc40:36667 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 613581fcdc40,36667,1732124821176 2024-11-20T17:47:03,031 DEBUG [RS:0;613581fcdc40:33139 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 613581fcdc40,33139,1732124820951 2024-11-20T17:47:03,031 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,36667,1732124821176' 2024-11-20T17:47:03,031 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,33139,1732124820951' 2024-11-20T17:47:03,031 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:47:03,031 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:47:03,032 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:47:03,032 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:47:03,033 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:47:03,033 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:47:03,033 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:47:03,033 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:47:03,033 DEBUG [RS:0;613581fcdc40:33139 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 613581fcdc40,33139,1732124820951 2024-11-20T17:47:03,033 DEBUG [RS:1;613581fcdc40:36667 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 613581fcdc40,36667,1732124821176 2024-11-20T17:47:03,033 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,33139,1732124820951' 2024-11-20T17:47:03,033 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,36667,1732124821176' 2024-11-20T17:47:03,033 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:47:03,034 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:47:03,034 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:47:03,034 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:47:03,035 DEBUG [RS:0;613581fcdc40:33139 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:47:03,035 DEBUG [RS:1;613581fcdc40:36667 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:47:03,035 INFO [RS:0;613581fcdc40:33139 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:47:03,035 INFO [RS:1;613581fcdc40:36667 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:47:03,035 INFO [RS:0;613581fcdc40:33139 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:47:03,035 INFO [RS:1;613581fcdc40:36667 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:47:03,140 INFO [RS:1;613581fcdc40:36667 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:47:03,140 INFO [RS:0;613581fcdc40:33139 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:47:03,141 INFO [RS:0;613581fcdc40:33139 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=613581fcdc40%2C33139%2C1732124820951, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,33139,1732124820951, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs, maxLogs=55 2024-11-20T17:47:03,142 INFO [RS:1;613581fcdc40:36667 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=613581fcdc40%2C36667%2C1732124821176, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,36667,1732124821176, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs, maxLogs=55 2024-11-20T17:47:03,167 INFO [RS:1;613581fcdc40:36667 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,36667,1732124821176/613581fcdc40%2C36667%2C1732124821176.1732124823147 2024-11-20T17:47:03,167 INFO [RS:0;613581fcdc40:33139 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,33139,1732124820951/613581fcdc40%2C33139%2C1732124820951.1732124823147 2024-11-20T17:47:03,167 DEBUG [RS:1;613581fcdc40:36667 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:03,167 DEBUG [RS:0;613581fcdc40:33139 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:03,221 DEBUG [613581fcdc40:37951 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T17:47:03,232 DEBUG [613581fcdc40:37951 {}] balancer.BalancerClusterState(204): Hosts are {613581fcdc40=0} racks are {/default-rack=0} 2024-11-20T17:47:03,236 DEBUG [613581fcdc40:37951 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T17:47:03,236 DEBUG [613581fcdc40:37951 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T17:47:03,236 DEBUG [613581fcdc40:37951 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:47:03,236 DEBUG [613581fcdc40:37951 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:47:03,236 INFO [613581fcdc40:37951 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:47:03,236 INFO [613581fcdc40:37951 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:47:03,236 DEBUG [613581fcdc40:37951 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T17:47:03,246 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=613581fcdc40,36667,1732124821176 2024-11-20T17:47:03,252 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,36667,1732124821176, state=OPENING 2024-11-20T17:47:03,274 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T17:47:03,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:03,285 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:03,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:03,286 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:03,286 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:03,286 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:03,290 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:47:03,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,36667,1732124821176}] 2024-11-20T17:47:03,478 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T17:47:03,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60833, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T17:47:03,492 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T17:47:03,493 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:03,494 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T17:47:03,496 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=613581fcdc40%2C36667%2C1732124821176.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,36667,1732124821176, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs, maxLogs=55 2024-11-20T17:47:03,521 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,36667,1732124821176/613581fcdc40%2C36667%2C1732124821176.meta.1732124823498.meta 2024-11-20T17:47:03,523 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:03,524 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:03,526 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T17:47:03,529 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T17:47:03,535 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T17:47:03,540 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T17:47:03,541 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:03,541 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T17:47:03,542 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T17:47:03,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T17:47:03,544 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:47:03,545 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:03,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:03,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T17:47:03,546 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:47:03,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:03,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:03,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T17:47:03,548 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:47:03,548 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:03,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:03,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T17:47:03,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:47:03,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:03,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:03,550 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:47:03,551 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:03,552 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:03,552 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:47:03,552 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:47:03,552 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:47:03,554 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:47:03,555 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60839321, jitterRate=-0.09342347085475922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:47:03,555 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T17:47:03,560 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732124823542Writing region info on filesystem at 1732124823543 (+1 ms)Initializing all the Stores at 1732124823543Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124823543Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124823543Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124823543Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124823543Cleaning up temporary data from old regions at 1732124823552 (+9 ms)Running coprocessor post-open hooks at 1732124823556 (+4 ms)Region opened successfully at 1732124823560 (+4 ms) 2024-11-20T17:47:03,566 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732124823466 2024-11-20T17:47:03,578 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T17:47:03,579 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T17:47:03,580 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,36667,1732124821176 2024-11-20T17:47:03,582 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,36667,1732124821176, state=OPEN 2024-11-20T17:47:03,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:03,632 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:03,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:03,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:03,633 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=613581fcdc40,36667,1732124821176 2024-11-20T17:47:03,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:03,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:03,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T17:47:03,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,36667,1732124821176 in 340 msec 2024-11-20T17:47:03,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T17:47:03,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 833 msec 2024-11-20T17:47:03,645 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:03,646 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T17:47:03,671 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:03,672 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,36667,1732124821176, seqNum=-1] 2024-11-20T17:47:03,696 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:03,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37705, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:03,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1430 sec 2024-11-20T17:47:03,718 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732124823717, completionTime=-1 2024-11-20T17:47:03,721 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T17:47:03,721 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:47:03,750 INFO [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:47:03,750 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732124883750 2024-11-20T17:47:03,750 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732124943750 2024-11-20T17:47:03,751 INFO [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-20T17:47:03,753 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:47:03,763 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37951,1732124819468-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,764 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37951,1732124819468-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,764 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37951,1732124819468-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,765 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-613581fcdc40:37951, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,766 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,766 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,772 DEBUG [master/613581fcdc40:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T17:47:03,796 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.382sec 2024-11-20T17:47:03,802 INFO [master/613581fcdc40:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T17:47:03,803 INFO [master/613581fcdc40:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T17:47:03,804 INFO [master/613581fcdc40:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T17:47:03,805 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T17:47:03,805 INFO [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T17:47:03,806 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37951,1732124819468-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:47:03,806 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37951,1732124819468-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T17:47:03,808 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T17:47:03,809 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T17:47:03,810 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37951,1732124819468-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:03,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:03,833 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T17:47:03,834 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T17:47:03,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,37951,-1 for getting cluster id 2024-11-20T17:47:03,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:03,852 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2282afdf-e043-4617-892c-c46cc1d3c615' 2024-11-20T17:47:03,855 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:03,855 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2282afdf-e043-4617-892c-c46cc1d3c615" 2024-11-20T17:47:03,856 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e389653, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:03,856 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,37951,-1] 2024-11-20T17:47:03,860 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:03,863 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:03,864 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:03,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:03,868 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:03,876 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,36667,1732124821176, seqNum=-1] 2024-11-20T17:47:03,877 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:03,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:03,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=613581fcdc40,37951,1732124819468 2024-11-20T17:47:03,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:47:03,927 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,37951,1732124819468 2024-11-20T17:47:03,930 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ecda439 2024-11-20T17:47:03,931 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:47:03,940 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:47:03,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732124823903', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:47:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732124823903 2024-11-20T17:47:03,958 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732124823903 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T17:47:03,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732124823903" procId is: 4 2024-11-20T17:47:03,960 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:03,962 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732124823903 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T17:47:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:03,980 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732124823903-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 12a55d5659a2829ba27c922dd4d95a5a, NAME => 'testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732124823903', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec 2024-11-20T17:47:03,987 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732124823903-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:03,987 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732124823903-pool-0 {}] regionserver.HRegion(1722): Closing 12a55d5659a2829ba27c922dd4d95a5a, disabling compactions & flushes 2024-11-20T17:47:03,987 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732124823903-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:03,987 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732124823903-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:03,988 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732124823903-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. after waiting 0 ms 2024-11-20T17:47:03,988 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732124823903-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:03,988 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732124823903-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:03,988 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732124823903-pool-0 {}] regionserver.HRegion(1676): Region close journal for 12a55d5659a2829ba27c922dd4d95a5a: Waiting for close lock at 1732124823987Disabling compacts and flushes for region at 1732124823987Disabling writes for close at 1732124823988 (+1 ms)Writing region close event to WAL at 1732124823988Closed at 1732124823988 2024-11-20T17:47:03,990 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732124823903 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T17:47:03,995 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732124823990"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732124823990"}]},"ts":"1732124823990"} 2024-11-20T17:47:03,999 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T17:47:04,001 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732124823903 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T17:47:04,004 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732124823903","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124824001"}]},"ts":"1732124824001"} 2024-11-20T17:47:04,009 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732124823903, state=ENABLING in hbase:meta 2024-11-20T17:47:04,009 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {613581fcdc40=0} racks are {/default-rack=0} 2024-11-20T17:47:04,010 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T17:47:04,010 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T17:47:04,010 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:47:04,010 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:47:04,010 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:47:04,010 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:47:04,010 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T17:47:04,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, ASSIGN}] 2024-11-20T17:47:04,014 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, ASSIGN 2024-11-20T17:47:04,015 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, ASSIGN; state=OFFLINE, location=613581fcdc40,33139,1732124820951; forceNewPlan=false, retain=false 2024-11-20T17:47:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:04,168 INFO [613581fcdc40:37951 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T17:47:04,169 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=12a55d5659a2829ba27c922dd4d95a5a, regionState=OPENING, regionLocation=613581fcdc40,33139,1732124820951 2024-11-20T17:47:04,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, ASSIGN because future has completed 2024-11-20T17:47:04,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12a55d5659a2829ba27c922dd4d95a5a, server=613581fcdc40,33139,1732124820951}] 2024-11-20T17:47:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:04,327 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T17:47:04,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48143, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T17:47:04,332 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:04,332 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 12a55d5659a2829ba27c922dd4d95a5a, NAME => 'testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:04,333 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732124823903 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,333 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:04,334 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,334 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,336 INFO [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,342 INFO [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12a55d5659a2829ba27c922dd4d95a5a columnFamilyName family 2024-11-20T17:47:04,342 DEBUG [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:04,343 INFO [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] regionserver.HStore(327): Store=12a55d5659a2829ba27c922dd4d95a5a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:04,343 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,345 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,345 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,345 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,345 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,346 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,352 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:47:04,353 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 12a55d5659a2829ba27c922dd4d95a5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73046662, jitterRate=0.08848008513450623}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:47:04,353 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:04,354 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 12a55d5659a2829ba27c922dd4d95a5a: Running coprocessor pre-open hook at 1732124824334Writing region info on filesystem at 1732124824334Initializing all the Stores at 1732124824335 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124824335Cleaning up temporary data from old regions at 1732124824345 (+10 ms)Running coprocessor post-open hooks at 1732124824353 (+8 ms)Region opened successfully at 1732124824353 2024-11-20T17:47:04,355 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a., pid=6, masterSystemTime=1732124824326 2024-11-20T17:47:04,358 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:04,358 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:04,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=12a55d5659a2829ba27c922dd4d95a5a, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,33139,1732124820951 2024-11-20T17:47:04,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12a55d5659a2829ba27c922dd4d95a5a, server=613581fcdc40,33139,1732124820951 because future has completed 2024-11-20T17:47:04,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T17:47:04,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 12a55d5659a2829ba27c922dd4d95a5a, server=613581fcdc40,33139,1732124820951 in 190 msec 2024-11-20T17:47:04,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T17:47:04,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, ASSIGN in 353 msec 2024-11-20T17:47:04,369 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732124823903 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T17:47:04,369 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732124823903","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124824369"}]},"ts":"1732124824369"} 2024-11-20T17:47:04,372 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732124823903, state=ENABLED in hbase:meta 2024-11-20T17:47:04,373 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732124823903 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T17:47:04,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732124823903 in 423 msec 2024-11-20T17:47:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:04,594 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732124823903 completed 2024-11-20T17:47:04,594 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732124823903 get assigned. Timeout = 60000ms 2024-11-20T17:47:04,597 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:47:04,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732124823903 assigned to meta. Checking AM states. 2024-11-20T17:47:04,605 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:47:04,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732124823903 assigned. 2024-11-20T17:47:04,614 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732124823903', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a., hostname=613581fcdc40,33139,1732124820951, seqNum=2] 2024-11-20T17:47:04,615 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:04,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:04,631 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 12a55d5659a2829ba27c922dd4d95a5a 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T17:47:04,661 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on da1259fab6de4a5cb67f55e8641436c2 2024-11-20T17:47:04,706 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/.tmp/family/da1259fab6de4a5cb67f55e8641436c2 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T17:47:04,712 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/.tmp/family/da1259fab6de4a5cb67f55e8641436c2 2024-11-20T17:47:04,744 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/.tmp/family/da1259fab6de4a5cb67f55e8641436c2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2 2024-11-20T17:47:04,745 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2, delay=903 ms 2024-11-20T17:47:04,746 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T17:47:04,753 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 12a55d5659a2829ba27c922dd4d95a5a in 121ms, sequenceid=6, compaction requested=false 2024-11-20T17:47:04,754 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732124823903' 2024-11-20T17:47:04,756 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 12a55d5659a2829ba27c922dd4d95a5a: 2024-11-20T17:47:05,651 DEBUG [hfile-prefetch-1732124824745 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key da1259fab6de4a5cb67f55e8641436c2_0. 2024-11-20T17:47:05,657 DEBUG [hfile-prefetch-1732124824745 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key da1259fab6de4a5cb67f55e8641436c2_117. 2024-11-20T17:47:05,659 DEBUG [hfile-prefetch-1732124824745 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2, with total blocks 2, and data blocks 1 2024-11-20T17:47:05,660 DEBUG [hfile-prefetch-1732124824745 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T17:47:05,660 DEBUG [hfile-prefetch-1732124824745 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T17:47:05,661 DEBUG [hfile-prefetch-1732124824745 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2. Will sleep 100ms and try the verification again. 2024-11-20T17:47:05,761 DEBUG [hfile-prefetch-1732124824745 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2, with total blocks 2, and data blocks 1 2024-11-20T17:47:05,761 DEBUG [hfile-prefetch-1732124824745 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T17:47:05,762 DEBUG [hfile-prefetch-1732124824745 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T17:47:05,762 DEBUG [hfile-prefetch-1732124824745 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2. Will sleep 100ms and try the verification again. 2024-11-20T17:47:05,771 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T17:47:05,771 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T17:47:05,772 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T17:47:05,772 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T17:47:05,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=12a55d5659a2829ba27c922dd4d95a5a, source=613581fcdc40,33139,1732124820951, destination=613581fcdc40,36667,1732124821176, warming up region on 613581fcdc40,36667,1732124821176 2024-11-20T17:47:05,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=12a55d5659a2829ba27c922dd4d95a5a, source=613581fcdc40,33139,1732124820951, destination=613581fcdc40,36667,1732124821176, running balancer 2024-11-20T17:47:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, REOPEN/MOVE 2024-11-20T17:47:05,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.HRegion(7855): Warmup {ENCODED => 12a55d5659a2829ba27c922dd4d95a5a, NAME => 'testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:05,779 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, REOPEN/MOVE 2024-11-20T17:47:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:05,780 INFO [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:05,781 INFO [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12a55d5659a2829ba27c922dd4d95a5a columnFamilyName family 2024-11-20T17:47:05,781 DEBUG [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:05,781 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=12a55d5659a2829ba27c922dd4d95a5a, regionState=CLOSING, regionLocation=613581fcdc40,33139,1732124820951 2024-11-20T17:47:05,784 DEBUG [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2 2024-11-20T17:47:05,784 INFO [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] regionserver.HStore(327): Store=12a55d5659a2829ba27c922dd4d95a5a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.HRegion(1722): Closing 12a55d5659a2829ba27c922dd4d95a5a, disabling compactions & flushes 2024-11-20T17:47:05,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. after waiting 0 ms 2024-11-20T17:47:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:05,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, REOPEN/MOVE because future has completed 2024-11-20T17:47:05,786 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-20T17:47:05,786 DEBUG [StoreFileCloser-12a55d5659a2829ba27c922dd4d95a5a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2 2024-11-20T17:47:05,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12a55d5659a2829ba27c922dd4d95a5a, server=613581fcdc40,33139,1732124820951}] 2024-11-20T17:47:05,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36667 {}] regionserver.HRegion(1676): Region close journal for 12a55d5659a2829ba27c922dd4d95a5a: Waiting for close lock at 1732124825784Disabling compacts and flushes for region at 1732124825784Disabling writes for close at 1732124825785 (+1 ms)Writing region close event to WAL at 1732124825786 (+1 ms)Closed at 1732124825786 2024-11-20T17:47:05,788 WARN [hfile-prefetch-1732124824745 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 10 more 2024-11-20T17:47:05,789 DEBUG [hfile-prefetch-1732124824745 {}] hfile.PrefetchExecutor(121): Prefetch completed for da1259fab6de4a5cb67f55e8641436c2 2024-11-20T17:47:05,791 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T17:47:05,791 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732124825771 2024-11-20T17:47:05,791 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T17:47:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37951 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T17:47:05,945 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:05,945 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-20T17:47:05,946 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 12a55d5659a2829ba27c922dd4d95a5a, disabling compactions & flushes 2024-11-20T17:47:05,946 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:05,946 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:05,946 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. after waiting 0 ms 2024-11-20T17:47:05,946 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:05,948 DEBUG [StoreFileCloser-12a55d5659a2829ba27c922dd4d95a5a-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file da1259fab6de4a5cb67f55e8641436c2, starting offset: 0, end offset: 9223372036854775807 2024-11-20T17:47:05,950 DEBUG [StoreFileCloser-12a55d5659a2829ba27c922dd4d95a5a-family-1 {}] bucket.BucketCache(856): removed key da1259fab6de4a5cb67f55e8641436c2_0 from back map with offset lock 2094080 in the evict process 2024-11-20T17:47:05,957 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T17:47:05,958 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:05,958 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 12a55d5659a2829ba27c922dd4d95a5a: Waiting for close lock at 1732124825946Running coprocessor pre-close hooks at 1732124825946Disabling compacts and flushes for region at 1732124825946Disabling writes for close at 1732124825946Writing region close event to WAL at 1732124825951 (+5 ms)Running coprocessor post-close hooks at 1732124825957 (+6 ms)Closed at 1732124825958 (+1 ms) 2024-11-20T17:47:05,958 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 12a55d5659a2829ba27c922dd4d95a5a move to 613581fcdc40,36667,1732124821176 record at close sequenceid=6 2024-11-20T17:47:05,962 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:05,962 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=12a55d5659a2829ba27c922dd4d95a5a, regionState=CLOSED 2024-11-20T17:47:05,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12a55d5659a2829ba27c922dd4d95a5a, server=613581fcdc40,33139,1732124820951 because future has completed 2024-11-20T17:47:05,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T17:47:05,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 12a55d5659a2829ba27c922dd4d95a5a, server=613581fcdc40,33139,1732124820951 in 180 msec 2024-11-20T17:47:05,970 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, REOPEN/MOVE; state=CLOSED, location=613581fcdc40,36667,1732124821176; forceNewPlan=false, retain=false 2024-11-20T17:47:05,991 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T17:47:05,991 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T17:47:05,992 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T17:47:05,992 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T17:47:05,995 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T17:47:05,995 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732124825991 2024-11-20T17:47:05,995 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T17:47:06,120 INFO [613581fcdc40:37951 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T17:47:06,121 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=12a55d5659a2829ba27c922dd4d95a5a, regionState=OPENING, regionLocation=613581fcdc40,36667,1732124821176 2024-11-20T17:47:06,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, REOPEN/MOVE because future has completed 2024-11-20T17:47:06,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12a55d5659a2829ba27c922dd4d95a5a, server=613581fcdc40,36667,1732124821176}] 2024-11-20T17:47:06,280 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:06,280 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 12a55d5659a2829ba27c922dd4d95a5a, NAME => 'testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:06,281 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732124823903 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,281 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:06,281 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,281 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,282 INFO [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,284 INFO [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12a55d5659a2829ba27c922dd4d95a5a columnFamilyName family 2024-11-20T17:47:06,284 DEBUG [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:06,286 DEBUG [StoreFileOpener-12a55d5659a2829ba27c922dd4d95a5a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2, delay=966 ms 2024-11-20T17:47:06,287 DEBUG [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2 2024-11-20T17:47:06,288 INFO [StoreOpener-12a55d5659a2829ba27c922dd4d95a5a-1 {}] regionserver.HStore(327): Store=12a55d5659a2829ba27c922dd4d95a5a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:06,288 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,289 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,289 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,289 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,289 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,294 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,295 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 12a55d5659a2829ba27c922dd4d95a5a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60330866, jitterRate=-0.10100004076957703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:47:06,295 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,296 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 12a55d5659a2829ba27c922dd4d95a5a: Running coprocessor pre-open hook at 1732124826282Writing region info on filesystem at 1732124826282Initializing all the Stores at 1732124826282Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124826282Cleaning up temporary data from old regions at 1732124826289 (+7 ms)Running coprocessor post-open hooks at 1732124826295 (+6 ms)Region opened successfully at 1732124826296 (+1 ms) 2024-11-20T17:47:06,297 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a., pid=9, masterSystemTime=1732124826277 2024-11-20T17:47:06,301 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:06,301 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:06,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=12a55d5659a2829ba27c922dd4d95a5a, regionState=OPEN, openSeqNum=10, regionLocation=613581fcdc40,36667,1732124821176 2024-11-20T17:47:06,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12a55d5659a2829ba27c922dd4d95a5a, server=613581fcdc40,36667,1732124821176 because future has completed 2024-11-20T17:47:06,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T17:47:06,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 12a55d5659a2829ba27c922dd4d95a5a, server=613581fcdc40,36667,1732124821176 in 183 msec 2024-11-20T17:47:06,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732124823903, region=12a55d5659a2829ba27c922dd4d95a5a, REOPEN/MOVE in 533 msec 2024-11-20T17:47:06,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T17:47:06,333 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:47:06,333 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:47:06,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:06,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:06,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:47:06,338 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T17:47:06,338 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=520236456, stopped=false 2024-11-20T17:47:06,338 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=613581fcdc40,37951,1732124819468 2024-11-20T17:47:06,390 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:06,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:06,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:06,391 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:06,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:06,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:06,391 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:47:06,393 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:06,394 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:06,394 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:47:06,394 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:06,394 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:47:06,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:06,395 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '613581fcdc40,33139,1732124820951' ***** 2024-11-20T17:47:06,395 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T17:47:06,395 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '613581fcdc40,36667,1732124821176' ***** 2024-11-20T17:47:06,395 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T17:47:06,395 INFO [RS:0;613581fcdc40:33139 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T17:47:06,396 INFO [RS:1;613581fcdc40:36667 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T17:47:06,396 INFO [RS:0;613581fcdc40:33139 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.persistence 2024-11-20T17:47:06,396 INFO [RS:1;613581fcdc40:36667 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/bucket.persistence 2024-11-20T17:47:06,396 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T17:47:06,396 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T17:47:06,396 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T17:47:06,396 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T17:47:06,396 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T17:47:06,396 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T17:47:06,396 INFO [RS:0;613581fcdc40:33139 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T17:47:06,396 INFO [RS:1;613581fcdc40:36667 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T17:47:06,397 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T17:47:06,397 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T17:47:06,397 DEBUG [RS:1;613581fcdc40:36667 {}] bucket.BucketCache(1487): Thread RS:1;613581fcdc40:36667 started persisting bucket cache to file 2024-11-20T17:47:06,397 DEBUG [RS:1;613581fcdc40:36667 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T17:47:06,397 DEBUG [RS:1;613581fcdc40:36667 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T17:47:06,401 DEBUG [RS:1;613581fcdc40:36667 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T17:47:06,401 DEBUG [RS:1;613581fcdc40:36667 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732124826397 2024-11-20T17:47:06,401 DEBUG [RS:1;613581fcdc40:36667 {}] bucket.BucketCache(1509): Thread RS:1;613581fcdc40:36667 finished persisting bucket cache to file, renaming 2024-11-20T17:47:06,401 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T17:47:06,402 INFO [RS:1;613581fcdc40:36667 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T17:47:06,402 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T17:47:06,402 INFO [RS:1;613581fcdc40:36667 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T17:47:06,402 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(3091): Received CLOSE for 12a55d5659a2829ba27c922dd4d95a5a 2024-11-20T17:47:06,403 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(959): stopping server 613581fcdc40,36667,1732124821176 2024-11-20T17:47:06,403 INFO [RS:1;613581fcdc40:36667 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:47:06,403 INFO [RS:1;613581fcdc40:36667 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;613581fcdc40:36667. 2024-11-20T17:47:06,403 DEBUG [RS:1;613581fcdc40:36667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:47:06,403 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T17:47:06,403 DEBUG [RS:1;613581fcdc40:36667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:06,403 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12a55d5659a2829ba27c922dd4d95a5a, disabling compactions & flushes 2024-11-20T17:47:06,403 INFO [RS:1;613581fcdc40:36667 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T17:47:06,403 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:06,403 INFO [RS:1;613581fcdc40:36667 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T17:47:06,403 INFO [RS:1;613581fcdc40:36667 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T17:47:06,403 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:06,404 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. after waiting 0 ms 2024-11-20T17:47:06,404 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T17:47:06,404 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:06,404 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T17:47:06,404 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T17:47:06,404 DEBUG [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 12a55d5659a2829ba27c922dd4d95a5a=testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a.} 2024-11-20T17:47:06,404 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:47:06,404 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:47:06,404 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:47:06,405 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:47:06,405 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:47:06,405 DEBUG [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(1351): Waiting on 12a55d5659a2829ba27c922dd4d95a5a, 1588230740 2024-11-20T17:47:06,405 DEBUG [StoreFileCloser-12a55d5659a2829ba27c922dd4d95a5a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/family/da1259fab6de4a5cb67f55e8641436c2 2024-11-20T17:47:06,405 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-20T17:47:06,407 DEBUG [RS:0;613581fcdc40:33139 {}] bucket.BucketCache(1487): Thread RS:0;613581fcdc40:33139 started persisting bucket cache to file 2024-11-20T17:47:06,407 DEBUG [RS:0;613581fcdc40:33139 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T17:47:06,407 DEBUG [RS:0;613581fcdc40:33139 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T17:47:06,409 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnRegionMove1732124823903/12a55d5659a2829ba27c922dd4d95a5a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T17:47:06,410 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:06,410 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12a55d5659a2829ba27c922dd4d95a5a: Waiting for close lock at 1732124826403Running coprocessor pre-close hooks at 1732124826403Disabling compacts and flushes for region at 1732124826403Disabling writes for close at 1732124826404 (+1 ms)Writing region close event to WAL at 1732124826405 (+1 ms)Running coprocessor post-close hooks at 1732124826410 (+5 ms)Closed at 1732124826410 2024-11-20T17:47:06,410 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a. 2024-11-20T17:47:06,411 DEBUG [RS:0;613581fcdc40:33139 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T17:47:06,411 DEBUG [RS:0;613581fcdc40:33139 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732124826407 2024-11-20T17:47:06,411 DEBUG [RS:0;613581fcdc40:33139 {}] bucket.BucketCache(1509): Thread RS:0;613581fcdc40:33139 finished persisting bucket cache to file, renaming 2024-11-20T17:47:06,411 INFO [RS:0;613581fcdc40:33139 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T17:47:06,411 INFO [RS:0;613581fcdc40:33139 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T17:47:06,411 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T17:47:06,411 INFO [RS:0;613581fcdc40:33139 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T17:47:06,411 INFO [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(959): stopping server 613581fcdc40,33139,1732124820951 2024-11-20T17:47:06,411 INFO [RS:0;613581fcdc40:33139 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:47:06,411 INFO [RS:0;613581fcdc40:33139 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;613581fcdc40:33139. 2024-11-20T17:47:06,411 DEBUG [RS:0;613581fcdc40:33139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:47:06,411 DEBUG [RS:0;613581fcdc40:33139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:06,412 INFO [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(976): stopping server 613581fcdc40,33139,1732124820951; all regions closed. 2024-11-20T17:47:06,416 DEBUG [RS:0;613581fcdc40:33139 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs 2024-11-20T17:47:06,416 INFO [RS:0;613581fcdc40:33139 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C33139%2C1732124820951:(num 1732124823147) 2024-11-20T17:47:06,416 DEBUG [RS:0;613581fcdc40:33139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:06,416 INFO [RS:0;613581fcdc40:33139 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:06,416 INFO [RS:0;613581fcdc40:33139 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:47:06,417 INFO [RS:0;613581fcdc40:33139 {}] hbase.ChoreService(370): Chore service for: regionserver/613581fcdc40:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T17:47:06,417 INFO [RS:0;613581fcdc40:33139 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T17:47:06,417 INFO [regionserver/613581fcdc40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:47:06,417 INFO [RS:0;613581fcdc40:33139 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T17:47:06,417 INFO [RS:0;613581fcdc40:33139 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T17:47:06,417 INFO [RS:0;613581fcdc40:33139 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:47:06,417 INFO [RS:0;613581fcdc40:33139 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33139 2024-11-20T17:47:06,417 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ec47d0bed58a4e6690e75cc6eec39760 2024-11-20T17:47:06,435 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/info/ec47d0bed58a4e6690e75cc6eec39760 is 205, key is testBlockEvictionOnRegionMove1732124823903,,1732124823942.12a55d5659a2829ba27c922dd4d95a5a./info:regioninfo/1732124826301/Put/seqid=0 2024-11-20T17:47:06,436 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/info/ec47d0bed58a4e6690e75cc6eec39760 2024-11-20T17:47:06,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/613581fcdc40,33139,1732124820951 2024-11-20T17:47:06,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:47:06,442 INFO [RS:0;613581fcdc40:33139 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:47:06,443 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [613581fcdc40,33139,1732124820951] 2024-11-20T17:47:06,447 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5ae504a3f74340a38e4867b8b30ff1e3 2024-11-20T17:47:06,462 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/ns/5ae504a3f74340a38e4867b8b30ff1e3 is 43, key is default/ns:d/1732124823703/Put/seqid=0 2024-11-20T17:47:06,462 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/ns/5ae504a3f74340a38e4867b8b30ff1e3 2024-11-20T17:47:06,463 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/613581fcdc40,33139,1732124820951 already deleted, retry=false 2024-11-20T17:47:06,463 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 613581fcdc40,33139,1732124820951 expired; onlineServers=1 2024-11-20T17:47:06,470 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 98429f99a57447869064010590c17ff7 2024-11-20T17:47:06,475 INFO [regionserver/613581fcdc40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:06,476 INFO [regionserver/613581fcdc40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:06,483 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/table/98429f99a57447869064010590c17ff7 is 78, key is testBlockEvictionOnRegionMove1732124823903/table:state/1732124824369/Put/seqid=0 2024-11-20T17:47:06,483 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/table/98429f99a57447869064010590c17ff7 2024-11-20T17:47:06,486 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/info/ec47d0bed58a4e6690e75cc6eec39760 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/ec47d0bed58a4e6690e75cc6eec39760 2024-11-20T17:47:06,487 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/ec47d0bed58a4e6690e75cc6eec39760, delay=952 ms 2024-11-20T17:47:06,488 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/ec47d0bed58a4e6690e75cc6eec39760, entries=15, sequenceid=15, filesize=7.6 K 2024-11-20T17:47:06,489 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/ns/5ae504a3f74340a38e4867b8b30ff1e3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/5ae504a3f74340a38e4867b8b30ff1e3 2024-11-20T17:47:06,490 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/5ae504a3f74340a38e4867b8b30ff1e3, delay=918 ms 2024-11-20T17:47:06,491 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/5ae504a3f74340a38e4867b8b30ff1e3, entries=2, sequenceid=15, filesize=5.0 K 2024-11-20T17:47:06,491 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/table/98429f99a57447869064010590c17ff7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/98429f99a57447869064010590c17ff7 2024-11-20T17:47:06,493 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/98429f99a57447869064010590c17ff7, delay=963 ms 2024-11-20T17:47:06,494 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/98429f99a57447869064010590c17ff7, entries=2, sequenceid=15, filesize=5.3 K 2024-11-20T17:47:06,495 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false 2024-11-20T17:47:06,495 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T17:47:06,496 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/ec47d0bed58a4e6690e75cc6eec39760 2024-11-20T17:47:06,496 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/5ae504a3f74340a38e4867b8b30ff1e3 2024-11-20T17:47:06,497 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/98429f99a57447869064010590c17ff7 2024-11-20T17:47:06,501 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T17:47:06,502 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T17:47:06,502 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:47:06,502 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732124826404Running coprocessor pre-close hooks at 1732124826404Disabling compacts and flushes for region at 1732124826404Disabling writes for close at 1732124826405 (+1 ms)Obtaining lock to block concurrent updates at 1732124826405Preparing flush snapshotting stores in 1588230740 at 1732124826405Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732124826406 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732124826406Flushing 1588230740/info: creating writer at 1732124826406Flushing 1588230740/info: appending metadata at 1732124826434 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732124826434Flushing 1588230740/ns: creating writer at 1732124826440 (+6 ms)Flushing 1588230740/ns: appending metadata at 1732124826461 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732124826461Flushing 1588230740/table: creating writer at 1732124826465 (+4 ms)Flushing 1588230740/table: appending metadata at 1732124826482 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732124826482Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@146a200: reopening flushed file at 1732124826485 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51b969e6: reopening flushed file at 1732124826488 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75fcf810: reopening flushed file at 1732124826491 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false at 1732124826495 (+4 ms)Writing region close event to WAL at 1732124826497 (+2 ms)Running coprocessor post-close hooks at 1732124826502 (+5 ms)Closed at 1732124826502 2024-11-20T17:47:06,503 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T17:47:06,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:06,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33139-0x101595b3a5c0001, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:06,553 INFO [RS:0;613581fcdc40:33139 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:47:06,554 INFO [RS:0;613581fcdc40:33139 {}] regionserver.HRegionServer(1031): Exiting; stopping=613581fcdc40,33139,1732124820951; zookeeper connection closed. 2024-11-20T17:47:06,554 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@95ac4e7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@95ac4e7 2024-11-20T17:47:06,605 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(976): stopping server 613581fcdc40,36667,1732124821176; all regions closed. 2024-11-20T17:47:06,608 DEBUG [RS:1;613581fcdc40:36667 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs 2024-11-20T17:47:06,608 INFO [RS:1;613581fcdc40:36667 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C36667%2C1732124821176.meta:.meta(num 1732124823498) 2024-11-20T17:47:06,610 DEBUG [RS:1;613581fcdc40:36667 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs 2024-11-20T17:47:06,611 INFO [RS:1;613581fcdc40:36667 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C36667%2C1732124821176:(num 1732124823147) 2024-11-20T17:47:06,611 DEBUG [RS:1;613581fcdc40:36667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:06,611 INFO [RS:1;613581fcdc40:36667 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:06,611 INFO [RS:1;613581fcdc40:36667 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:47:06,611 INFO [RS:1;613581fcdc40:36667 {}] hbase.ChoreService(370): Chore service for: regionserver/613581fcdc40:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T17:47:06,612 INFO [regionserver/613581fcdc40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:47:06,611 INFO [RS:1;613581fcdc40:36667 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:47:06,612 INFO [RS:1;613581fcdc40:36667 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36667 2024-11-20T17:47:06,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:47:06,621 INFO [RS:1;613581fcdc40:36667 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:47:06,621 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/613581fcdc40,36667,1732124821176 2024-11-20T17:47:06,632 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [613581fcdc40,36667,1732124821176] 2024-11-20T17:47:06,642 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/613581fcdc40,36667,1732124821176 already deleted, retry=false 2024-11-20T17:47:06,642 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 613581fcdc40,36667,1732124821176 expired; onlineServers=0 2024-11-20T17:47:06,642 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '613581fcdc40,37951,1732124819468' ***** 2024-11-20T17:47:06,642 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T17:47:06,643 INFO [M:0;613581fcdc40:37951 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:47:06,643 INFO [M:0;613581fcdc40:37951 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:47:06,643 DEBUG [M:0;613581fcdc40:37951 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T17:47:06,643 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T17:47:06,643 DEBUG [M:0;613581fcdc40:37951 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T17:47:06,643 DEBUG [master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732124822716 {}] cleaner.HFileCleaner(306): Exit Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732124822716,5,FailOnTimeoutGroup] 2024-11-20T17:47:06,643 DEBUG [master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732124822715 {}] cleaner.HFileCleaner(306): Exit Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732124822715,5,FailOnTimeoutGroup] 2024-11-20T17:47:06,643 INFO [M:0;613581fcdc40:37951 {}] hbase.ChoreService(370): Chore service for: master/613581fcdc40:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T17:47:06,644 INFO [M:0;613581fcdc40:37951 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:47:06,644 DEBUG [M:0;613581fcdc40:37951 {}] master.HMaster(1795): Stopping service threads 2024-11-20T17:47:06,644 INFO [M:0;613581fcdc40:37951 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T17:47:06,644 INFO [M:0;613581fcdc40:37951 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:47:06,645 INFO [M:0;613581fcdc40:37951 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:47:06,645 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T17:47:06,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T17:47:06,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:06,653 DEBUG [M:0;613581fcdc40:37951 {}] zookeeper.ZKUtil(347): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T17:47:06,653 WARN [M:0;613581fcdc40:37951 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T17:47:06,654 INFO [M:0;613581fcdc40:37951 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/.lastflushedseqids 2024-11-20T17:47:06,668 INFO [M:0;613581fcdc40:37951 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:47:06,669 INFO [M:0;613581fcdc40:37951 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T17:47:06,669 DEBUG [M:0;613581fcdc40:37951 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T17:47:06,669 INFO [M:0;613581fcdc40:37951 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:06,669 DEBUG [M:0;613581fcdc40:37951 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:06,669 DEBUG [M:0;613581fcdc40:37951 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T17:47:06,669 DEBUG [M:0;613581fcdc40:37951 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:06,669 INFO [M:0;613581fcdc40:37951 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-20T17:47:06,680 DEBUG [M:0;613581fcdc40:37951 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b796c1d956f0448badb233cc4b5b5310 2024-11-20T17:47:06,700 DEBUG [M:0;613581fcdc40:37951 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b796c1d956f0448badb233cc4b5b5310 is 82, key is hbase:meta,,1/info:regioninfo/1732124823579/Put/seqid=0 2024-11-20T17:47:06,702 INFO [M:0;613581fcdc40:37951 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b796c1d956f0448badb233cc4b5b5310 2024-11-20T17:47:06,712 DEBUG [M:0;613581fcdc40:37951 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c2166489a5cc4229b6546f1b739d0143 2024-11-20T17:47:06,732 DEBUG [M:0;613581fcdc40:37951 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2166489a5cc4229b6546f1b739d0143 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732124824375/Put/seqid=0 2024-11-20T17:47:06,732 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:06,732 INFO [RS:1;613581fcdc40:36667 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:47:06,732 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36667-0x101595b3a5c0002, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:06,732 INFO [RS:1;613581fcdc40:36667 {}] regionserver.HRegionServer(1031): Exiting; stopping=613581fcdc40,36667,1732124821176; zookeeper connection closed. 2024-11-20T17:47:06,733 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73ea02d2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73ea02d2 2024-11-20T17:47:06,733 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-20T17:47:06,734 INFO [M:0;613581fcdc40:37951 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2166489a5cc4229b6546f1b739d0143 2024-11-20T17:47:06,754 DEBUG [M:0;613581fcdc40:37951 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a4e3e64f99804e158070ddce56e8e6b0 2024-11-20T17:47:06,769 DEBUG [M:0;613581fcdc40:37951 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a4e3e64f99804e158070ddce56e8e6b0 is 69, key is 613581fcdc40,33139,1732124820951/rs:state/1732124822819/Put/seqid=0 2024-11-20T17:47:06,769 INFO [M:0;613581fcdc40:37951 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a4e3e64f99804e158070ddce56e8e6b0 2024-11-20T17:47:06,771 DEBUG [M:0;613581fcdc40:37951 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b796c1d956f0448badb233cc4b5b5310 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b796c1d956f0448badb233cc4b5b5310 2024-11-20T17:47:06,774 INFO [M:0;613581fcdc40:37951 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b796c1d956f0448badb233cc4b5b5310, entries=8, sequenceid=84, filesize=5.5 K 2024-11-20T17:47:06,774 DEBUG [M:0;613581fcdc40:37951 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2166489a5cc4229b6546f1b739d0143 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2166489a5cc4229b6546f1b739d0143 2024-11-20T17:47:06,776 INFO [M:0;613581fcdc40:37951 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2166489a5cc4229b6546f1b739d0143, entries=9, sequenceid=84, filesize=6.5 K 2024-11-20T17:47:06,776 DEBUG [M:0;613581fcdc40:37951 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a4e3e64f99804e158070ddce56e8e6b0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a4e3e64f99804e158070ddce56e8e6b0 2024-11-20T17:47:06,778 INFO [M:0;613581fcdc40:37951 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a4e3e64f99804e158070ddce56e8e6b0, entries=2, sequenceid=84, filesize=5.1 K 2024-11-20T17:47:06,780 INFO [M:0;613581fcdc40:37951 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=84, compaction requested=false 2024-11-20T17:47:06,789 INFO [M:0;613581fcdc40:37951 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:06,789 DEBUG [M:0;613581fcdc40:37951 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732124826669Disabling compacts and flushes for region at 1732124826669Disabling writes for close at 1732124826669Obtaining lock to block concurrent updates at 1732124826669Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732124826669Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732124826670 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732124826670Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732124826670Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732124826699 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732124826699Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732124826704 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732124826731 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732124826732 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732124826737 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732124826768 (+31 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732124826768Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22f62144: reopening flushed file at 1732124826771 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c4f835b: reopening flushed file at 1732124826774 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12f03042: reopening flushed file at 1732124826776 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=84, compaction requested=false at 1732124826780 (+4 ms)Writing region close event to WAL at 1732124826788 (+8 ms)Closed at 1732124826789 (+1 ms) 2024-11-20T17:47:06,791 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:47:06,791 INFO [M:0;613581fcdc40:37951 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T17:47:06,792 INFO [M:0;613581fcdc40:37951 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37951 2024-11-20T17:47:06,792 INFO [M:0;613581fcdc40:37951 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:47:06,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:06,900 INFO [M:0;613581fcdc40:37951 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:47:06,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37951-0x101595b3a5c0000, quorum=127.0.0.1:59586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:06,906 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T17:47:06,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T17:47:06,929 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:06,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732124826286 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732124824745 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732124826490 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/613581fcdc40:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/613581fcdc40:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/613581fcdc40:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732124826487 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/613581fcdc40:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=504 (was 495) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9460 (was 10129) 2024-11-20T17:47:06,952 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=504, ProcessCount=11, AvailableMemoryMB=9458 2024-11-20T17:47:06,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/hadoop-log-dir so I do NOT create it in target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1 2024-11-20T17:47:06,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/hadoop-tmp-dir so I do NOT create it in target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1 2024-11-20T17:47:06,952 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:06,959 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/cluster_4fb5671d-22da-fbd4-8778-b32978cf967e, deleteOnExit=true 2024-11-20T17:47:06,965 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/cluster_4fb5671d-22da-fbd4-8778-b32978cf967e/zookeeper_0, clientPort=56636, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/cluster_4fb5671d-22da-fbd4-8778-b32978cf967e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/cluster_4fb5671d-22da-fbd4-8778-b32978cf967e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T17:47:06,966 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56636 2024-11-20T17:47:06,987 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec with version=8 2024-11-20T17:47:06,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/hbase-staging 2024-11-20T17:47:06,991 INFO [Time-limited test {}] client.ConnectionUtils(128): master/613581fcdc40:0 server-side Connection retries=45 2024-11-20T17:47:06,991 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:06,991 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:06,991 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:47:06,991 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:06,991 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:47:06,991 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T17:47:06,991 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:47:06,992 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37693 2024-11-20T17:47:06,994 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37693 connecting to ZooKeeper ensemble=127.0.0.1:56636 2024-11-20T17:47:07,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:376930x0, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:47:07,050 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37693-0x101595b58fd0000 connected 2024-11-20T17:47:07,136 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:07,136 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:07,140 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:07,140 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec, hbase.cluster.distributed=false 2024-11-20T17:47:07,143 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:47:07,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37693 2024-11-20T17:47:07,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37693 2024-11-20T17:47:07,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37693 2024-11-20T17:47:07,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37693 2024-11-20T17:47:07,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37693 2024-11-20T17:47:07,164 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/613581fcdc40:0 server-side Connection retries=45 2024-11-20T17:47:07,164 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:07,164 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:07,165 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:47:07,165 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:07,165 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:47:07,165 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:47:07,165 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:47:07,166 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43195 2024-11-20T17:47:07,169 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43195 connecting to ZooKeeper ensemble=127.0.0.1:56636 2024-11-20T17:47:07,169 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:07,169 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:07,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431950x0, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:47:07,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43195-0x101595b58fd0001 connected 2024-11-20T17:47:07,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:07,179 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:47:07,180 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.cache 2024-11-20T17:47:07,180 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T17:47:07,180 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T17:47:07,180 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T17:47:07,182 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T17:47:07,182 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T17:47:07,183 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T17:47:07,183 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.persistence is complete. 2024-11-20T17:47:07,184 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T17:47:07,188 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:47:07,189 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T17:47:07,191 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:47:07,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43195 2024-11-20T17:47:07,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43195 2024-11-20T17:47:07,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43195 2024-11-20T17:47:07,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43195 2024-11-20T17:47:07,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43195 2024-11-20T17:47:07,219 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/613581fcdc40:0 server-side Connection retries=45 2024-11-20T17:47:07,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:07,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:07,220 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:47:07,220 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:07,220 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:47:07,220 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:47:07,220 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:47:07,222 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45773 2024-11-20T17:47:07,224 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45773 connecting to ZooKeeper ensemble=127.0.0.1:56636 2024-11-20T17:47:07,224 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:07,224 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:07,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:457730x0, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:47:07,239 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:457730x0, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:07,239 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45773-0x101595b58fd0002 connected 2024-11-20T17:47:07,240 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:47:07,240 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.cache 2024-11-20T17:47:07,240 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T17:47:07,240 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T17:47:07,240 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T17:47:07,244 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T17:47:07,244 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T17:47:07,244 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T17:47:07,245 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T17:47:07,245 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.persistence is complete. 2024-11-20T17:47:07,245 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:47:07,245 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T17:47:07,247 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:47:07,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45773 2024-11-20T17:47:07,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45773 2024-11-20T17:47:07,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45773 2024-11-20T17:47:07,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45773 2024-11-20T17:47:07,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45773 2024-11-20T17:47:07,272 DEBUG [M:0;613581fcdc40:37693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;613581fcdc40:37693 2024-11-20T17:47:07,272 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/613581fcdc40,37693,1732124826990 2024-11-20T17:47:07,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:07,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:07,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:07,284 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/613581fcdc40,37693,1732124826990 2024-11-20T17:47:07,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T17:47:07,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T17:47:07,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,294 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T17:47:07,294 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/613581fcdc40,37693,1732124826990 from backup master directory 2024-11-20T17:47:07,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:07,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:07,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/613581fcdc40,37693,1732124826990 2024-11-20T17:47:07,304 WARN [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:47:07,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:07,304 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=613581fcdc40,37693,1732124826990 2024-11-20T17:47:07,305 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/hbase.id] with ID: 57186c09-30f7-4530-a0c7-f0ca75a2036f 2024-11-20T17:47:07,305 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/.tmp/hbase.id 2024-11-20T17:47:07,308 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/hbase.id] 2024-11-20T17:47:07,344 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:07,345 INFO [master/613581fcdc40:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T17:47:07,347 INFO [master/613581fcdc40:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-20T17:47:07,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,363 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:47:07,364 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T17:47:07,364 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:07,386 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store 2024-11-20T17:47:07,394 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:07,395 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T17:47:07,395 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:07,395 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:07,395 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T17:47:07,395 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:07,395 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:07,395 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732124827394Disabling compacts and flushes for region at 1732124827394Disabling writes for close at 1732124827395 (+1 ms)Writing region close event to WAL at 1732124827395Closed at 1732124827395 2024-11-20T17:47:07,395 WARN [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/.initializing 2024-11-20T17:47:07,395 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/WALs/613581fcdc40,37693,1732124826990 2024-11-20T17:47:07,401 INFO [master/613581fcdc40:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:47:07,411 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=613581fcdc40%2C37693%2C1732124826990, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/WALs/613581fcdc40,37693,1732124826990, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/oldWALs, maxLogs=10 2024-11-20T17:47:07,430 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/WALs/613581fcdc40,37693,1732124826990/613581fcdc40%2C37693%2C1732124826990.1732124827411 2024-11-20T17:47:07,430 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:07,430 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:07,430 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:07,431 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,431 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T17:47:07,437 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:07,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:07,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T17:47:07,444 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:07,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:07,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T17:47:07,454 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:07,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:07,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,460 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T17:47:07,460 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:07,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:07,461 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,462 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,462 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,462 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,462 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,462 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:47:07,463 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:07,472 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:47:07,473 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69141677, jitterRate=0.030291274189949036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:47:07,474 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732124827431Initializing all the Stores at 1732124827431Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124827432 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124827432Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124827432Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124827432Cleaning up temporary data from old regions at 1732124827462 (+30 ms)Region opened successfully at 1732124827474 (+12 ms) 2024-11-20T17:47:07,475 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T17:47:07,479 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533a0924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:47:07,481 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T17:47:07,481 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T17:47:07,481 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T17:47:07,481 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:47:07,482 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:47:07,483 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-20T17:47:07,483 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T17:47:07,485 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T17:47:07,487 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T17:47:07,504 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T17:47:07,504 INFO [master/613581fcdc40:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T17:47:07,505 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T17:47:07,514 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T17:47:07,515 INFO [master/613581fcdc40:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T17:47:07,517 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T17:47:07,525 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T17:47:07,526 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T17:47:07,535 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T17:47:07,538 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T17:47:07,546 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T17:47:07,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:07,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:07,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:07,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,558 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=613581fcdc40,37693,1732124826990, sessionid=0x101595b58fd0000, setting cluster-up flag (Was=false) 2024-11-20T17:47:07,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,609 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T17:47:07,611 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=613581fcdc40,37693,1732124826990 2024-11-20T17:47:07,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:07,662 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T17:47:07,663 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=613581fcdc40,37693,1732124826990 2024-11-20T17:47:07,672 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T17:47:07,674 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:07,674 INFO [master/613581fcdc40:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T17:47:07,674 INFO [master/613581fcdc40:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T17:47:07,675 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 613581fcdc40,37693,1732124826990 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T17:47:07,677 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:07,677 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:07,677 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:07,677 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:07,677 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/613581fcdc40:0, corePoolSize=10, maxPoolSize=10 2024-11-20T17:47:07,677 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,677 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:47:07,678 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,682 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:07,682 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T17:47:07,682 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:07,683 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T17:47:07,696 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732124857696 2024-11-20T17:47:07,696 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T17:47:07,697 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T17:47:07,697 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T17:47:07,697 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T17:47:07,697 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T17:47:07,697 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T17:47:07,697 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,701 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T17:47:07,701 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T17:47:07,701 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T17:47:07,705 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T17:47:07,705 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T17:47:07,705 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732124827705,5,FailOnTimeoutGroup] 2024-11-20T17:47:07,710 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732124827705,5,FailOnTimeoutGroup] 2024-11-20T17:47:07,710 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,711 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T17:47:07,711 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,711 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,717 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T17:47:07,717 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec 2024-11-20T17:47:07,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:07,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T17:47:07,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:47:07,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:07,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:07,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T17:47:07,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:47:07,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:07,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:07,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T17:47:07,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:47:07,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:07,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:07,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T17:47:07,755 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:47:07,755 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:07,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:07,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:47:07,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:07,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:07,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:47:07,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:47:07,757 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:47:07,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:47:07,758 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(746): ClusterId : 57186c09-30f7-4530-a0c7-f0ca75a2036f 2024-11-20T17:47:07,758 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:47:07,770 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(746): ClusterId : 57186c09-30f7-4530-a0c7-f0ca75a2036f 2024-11-20T17:47:07,770 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:47:07,770 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:47:07,770 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:47:07,772 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:47:07,773 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65813190, jitterRate=-0.019307047128677368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:47:07,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732124827731Initializing all the Stores at 1732124827731Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124827731Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124827731Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124827731Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124827731Cleaning up temporary data from old regions at 1732124827757 (+26 ms)Region opened successfully at 1732124827773 (+16 ms) 2024-11-20T17:47:07,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:47:07,774 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:47:07,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:47:07,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:47:07,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:47:07,776 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:47:07,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732124827773Disabling compacts and flushes for region at 1732124827773Disabling writes for close at 1732124827774 (+1 ms)Writing region close event to WAL at 1732124827776 (+2 ms)Closed at 1732124827776 2024-11-20T17:47:07,777 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:07,777 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T17:47:07,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T17:47:07,779 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:47:07,780 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T17:47:07,789 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:47:07,789 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:47:07,790 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:47:07,790 DEBUG [RS:0;613581fcdc40:43195 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1959165e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:47:07,800 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:47:07,800 DEBUG [RS:1;613581fcdc40:45773 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c364f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:47:07,819 DEBUG [RS:0;613581fcdc40:43195 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;613581fcdc40:43195 2024-11-20T17:47:07,819 DEBUG [RS:1;613581fcdc40:45773 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;613581fcdc40:45773 2024-11-20T17:47:07,819 INFO [RS:0;613581fcdc40:43195 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:47:07,819 INFO [RS:0;613581fcdc40:43195 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:47:07,819 INFO [RS:1;613581fcdc40:45773 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:47:07,819 DEBUG [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:47:07,819 INFO [RS:1;613581fcdc40:45773 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:47:07,819 DEBUG [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:47:07,820 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(2659): reportForDuty to master=613581fcdc40,37693,1732124826990 with port=43195, startcode=1732124827163 2024-11-20T17:47:07,820 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(2659): reportForDuty to master=613581fcdc40,37693,1732124826990 with port=45773, startcode=1732124827218 2024-11-20T17:47:07,821 DEBUG [RS:0;613581fcdc40:43195 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:47:07,821 DEBUG [RS:1;613581fcdc40:45773 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:47:07,823 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33363, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:47:07,824 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37693 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 613581fcdc40,43195,1732124827163 2024-11-20T17:47:07,824 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37693 {}] master.ServerManager(517): Registering regionserver=613581fcdc40,43195,1732124827163 2024-11-20T17:47:07,825 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57413, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:47:07,826 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37693 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 613581fcdc40,45773,1732124827218 2024-11-20T17:47:07,826 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37693 {}] master.ServerManager(517): Registering regionserver=613581fcdc40,45773,1732124827218 2024-11-20T17:47:07,827 DEBUG [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec 2024-11-20T17:47:07,827 DEBUG [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T17:47:07,827 DEBUG [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:47:07,828 DEBUG [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec 2024-11-20T17:47:07,829 DEBUG [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T17:47:07,829 DEBUG [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:47:07,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:47:07,915 DEBUG [RS:0;613581fcdc40:43195 {}] zookeeper.ZKUtil(111): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/613581fcdc40,43195,1732124827163 2024-11-20T17:47:07,915 WARN [RS:0;613581fcdc40:43195 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:47:07,915 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [613581fcdc40,45773,1732124827218] 2024-11-20T17:47:07,915 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [613581fcdc40,43195,1732124827163] 2024-11-20T17:47:07,915 INFO [RS:0;613581fcdc40:43195 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:07,916 DEBUG [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163 2024-11-20T17:47:07,916 DEBUG [RS:1;613581fcdc40:45773 {}] zookeeper.ZKUtil(111): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/613581fcdc40,45773,1732124827218 2024-11-20T17:47:07,916 WARN [RS:1;613581fcdc40:45773 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:47:07,916 INFO [RS:1;613581fcdc40:45773 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:07,916 DEBUG [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,45773,1732124827218 2024-11-20T17:47:07,920 INFO [RS:1;613581fcdc40:45773 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:47:07,921 INFO [RS:0;613581fcdc40:43195 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:47:07,923 INFO [RS:1;613581fcdc40:45773 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:47:07,923 INFO [RS:1;613581fcdc40:45773 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:47:07,923 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,924 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:47:07,924 INFO [RS:1;613581fcdc40:45773 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:47:07,924 INFO [RS:0;613581fcdc40:43195 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:47:07,925 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,925 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,925 INFO [RS:0;613581fcdc40:43195 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:47:07,925 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,925 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,925 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,925 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,925 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,925 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:47:07,925 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:47:07,926 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,926 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,926 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,926 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,926 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,926 INFO [RS:0;613581fcdc40:43195 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:47:07,926 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,926 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:07,926 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,926 DEBUG [RS:1;613581fcdc40:45773 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:07,926 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,927 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,927 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,927 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,927 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,927 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,927 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,927 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,927 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,927 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:47:07,927 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,927 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,927 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,45773,1732124827218-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:47:07,927 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,928 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,928 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,928 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,928 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:07,928 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:07,928 DEBUG [RS:0;613581fcdc40:43195 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:07,929 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,929 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,929 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,929 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,929 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,929 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,43195,1732124827163-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:47:07,930 WARN [613581fcdc40:37693 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:47:07,943 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:47:07,943 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,43195,1732124827163-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,943 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,944 INFO [RS:0;613581fcdc40:43195 {}] regionserver.Replication(171): 613581fcdc40,43195,1732124827163 started 2024-11-20T17:47:07,944 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:47:07,944 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,45773,1732124827218-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,944 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,944 INFO [RS:1;613581fcdc40:45773 {}] regionserver.Replication(171): 613581fcdc40,45773,1732124827218 started 2024-11-20T17:47:07,957 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,957 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(1482): Serving as 613581fcdc40,43195,1732124827163, RpcServer on 613581fcdc40/172.17.0.2:43195, sessionid=0x101595b58fd0001 2024-11-20T17:47:07,957 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:47:07,957 DEBUG [RS:0;613581fcdc40:43195 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 613581fcdc40,43195,1732124827163 2024-11-20T17:47:07,957 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,43195,1732124827163' 2024-11-20T17:47:07,957 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:47:07,958 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:47:07,958 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:47:07,959 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:47:07,959 DEBUG [RS:0;613581fcdc40:43195 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 613581fcdc40,43195,1732124827163 2024-11-20T17:47:07,959 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,43195,1732124827163' 2024-11-20T17:47:07,959 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:47:07,959 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:47:07,960 DEBUG [RS:0;613581fcdc40:43195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:47:07,960 INFO [RS:0;613581fcdc40:43195 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:47:07,960 INFO [RS:0;613581fcdc40:43195 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:47:07,962 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:07,962 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(1482): Serving as 613581fcdc40,45773,1732124827218, RpcServer on 613581fcdc40/172.17.0.2:45773, sessionid=0x101595b58fd0002 2024-11-20T17:47:07,962 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:47:07,962 DEBUG [RS:1;613581fcdc40:45773 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 613581fcdc40,45773,1732124827218 2024-11-20T17:47:07,962 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,45773,1732124827218' 2024-11-20T17:47:07,962 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:47:07,963 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:47:07,964 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:47:07,964 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:47:07,964 DEBUG [RS:1;613581fcdc40:45773 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 613581fcdc40,45773,1732124827218 2024-11-20T17:47:07,964 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,45773,1732124827218' 2024-11-20T17:47:07,964 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:47:07,964 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:47:07,965 DEBUG [RS:1;613581fcdc40:45773 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:47:07,965 INFO [RS:1;613581fcdc40:45773 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:47:07,965 INFO [RS:1;613581fcdc40:45773 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:47:08,061 INFO [RS:0;613581fcdc40:43195 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:47:08,062 INFO [RS:0;613581fcdc40:43195 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=613581fcdc40%2C43195%2C1732124827163, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs, maxLogs=55 2024-11-20T17:47:08,066 INFO [RS:1;613581fcdc40:45773 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:47:08,066 INFO [RS:1;613581fcdc40:45773 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=613581fcdc40%2C45773%2C1732124827218, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,45773,1732124827218, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs, maxLogs=55 2024-11-20T17:47:08,088 INFO [RS:1;613581fcdc40:45773 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,45773,1732124827218/613581fcdc40%2C45773%2C1732124827218.1732124828067 2024-11-20T17:47:08,088 INFO [RS:0;613581fcdc40:43195 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163/613581fcdc40%2C43195%2C1732124827163.1732124828065 2024-11-20T17:47:08,089 DEBUG [RS:1;613581fcdc40:45773 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:08,091 DEBUG [RS:0;613581fcdc40:43195 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:08,180 DEBUG [613581fcdc40:37693 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T17:47:08,181 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(204): Hosts are {613581fcdc40=0} racks are {/default-rack=0} 2024-11-20T17:47:08,182 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T17:47:08,182 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T17:47:08,182 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:47:08,182 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:47:08,182 INFO [613581fcdc40:37693 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:47:08,182 INFO [613581fcdc40:37693 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:47:08,182 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T17:47:08,182 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=613581fcdc40,43195,1732124827163 2024-11-20T17:47:08,184 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,43195,1732124827163, state=OPENING 2024-11-20T17:47:08,198 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T17:47:08,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:08,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:08,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:08,209 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:47:08,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:08,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,43195,1732124827163}] 2024-11-20T17:47:08,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:08,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:08,363 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T17:47:08,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49103, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T17:47:08,372 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T17:47:08,372 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:08,373 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T17:47:08,374 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=613581fcdc40%2C43195%2C1732124827163.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs, maxLogs=55 2024-11-20T17:47:08,395 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163/613581fcdc40%2C43195%2C1732124827163.meta.1732124828374.meta 2024-11-20T17:47:08,395 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:08,396 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:08,396 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T17:47:08,396 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T17:47:08,397 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T17:47:08,397 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T17:47:08,397 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:08,397 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T17:47:08,397 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T17:47:08,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T17:47:08,399 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:47:08,399 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:08,400 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:08,400 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T17:47:08,401 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:47:08,401 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:08,401 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:08,401 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T17:47:08,402 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:47:08,402 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:08,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:08,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T17:47:08,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:47:08,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:08,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:08,404 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:47:08,405 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:08,405 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:08,405 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:47:08,405 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:47:08,406 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:47:08,406 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:47:08,407 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64233144, jitterRate=-0.04285156726837158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:47:08,407 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T17:47:08,407 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732124828397Writing region info on filesystem at 1732124828397Initializing all the Stores at 1732124828398 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124828398Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124828398Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124828398Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124828398Cleaning up temporary data from old regions at 1732124828405 (+7 ms)Running coprocessor post-open hooks at 1732124828407 (+2 ms)Region opened successfully at 1732124828407 2024-11-20T17:47:08,408 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732124828363 2024-11-20T17:47:08,412 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T17:47:08,412 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,43195,1732124827163 2024-11-20T17:47:08,412 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T17:47:08,413 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,43195,1732124827163, state=OPEN 2024-11-20T17:47:08,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:08,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:08,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:08,460 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=613581fcdc40,43195,1732124827163 2024-11-20T17:47:08,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:08,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:08,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:08,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T17:47:08,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,43195,1732124827163 in 250 msec 2024-11-20T17:47:08,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T17:47:08,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 685 msec 2024-11-20T17:47:08,465 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:08,465 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T17:47:08,468 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:08,469 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,43195,1732124827163, seqNum=-1] 2024-11-20T17:47:08,469 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:08,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37123, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:08,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 802 msec 2024-11-20T17:47:08,476 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732124828476, completionTime=-1 2024-11-20T17:47:08,476 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T17:47:08,476 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:47:08,479 INFO [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:47:08,479 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732124888479 2024-11-20T17:47:08,479 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732124948479 2024-11-20T17:47:08,479 INFO [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-20T17:47:08,479 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37693,1732124826990-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:08,480 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37693,1732124826990-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:08,480 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37693,1732124826990-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:08,480 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-613581fcdc40:37693, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:08,480 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:08,480 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:08,483 DEBUG [master/613581fcdc40:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T17:47:08,485 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.181sec 2024-11-20T17:47:08,485 INFO [master/613581fcdc40:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T17:47:08,486 INFO [master/613581fcdc40:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T17:47:08,486 INFO [master/613581fcdc40:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T17:47:08,486 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T17:47:08,486 INFO [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T17:47:08,486 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37693,1732124826990-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:47:08,486 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37693,1732124826990-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T17:47:08,487 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T17:47:08,487 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T17:47:08,487 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,37693,1732124826990-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:08,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:08,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,37693,-1 for getting cluster id 2024-11-20T17:47:08,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:08,564 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '57186c09-30f7-4530-a0c7-f0ca75a2036f' 2024-11-20T17:47:08,566 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:08,566 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "57186c09-30f7-4530-a0c7-f0ca75a2036f" 2024-11-20T17:47:08,567 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511ed698, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:08,568 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,37693,-1] 2024-11-20T17:47:08,568 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:08,569 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:08,572 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:08,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d796be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:08,574 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:08,575 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,43195,1732124827163, seqNum=-1] 2024-11-20T17:47:08,576 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:08,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:08,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=613581fcdc40,37693,1732124826990 2024-11-20T17:47:08,580 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:47:08,582 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,37693,1732124826990 2024-11-20T17:47:08,582 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@529cf03b 2024-11-20T17:47:08,582 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:47:08,584 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:47:08,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37693 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732124828580', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:47:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37693 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732124828580 2024-11-20T17:47:08,587 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732124828580 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T17:47:08,587 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:08,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37693 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732124828580" procId is: 4 2024-11-20T17:47:08,588 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732124828580 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T17:47:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:08,601 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732124828580-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eaba082561ecbbfc53fef1900b914756, NAME => 'testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732124828580', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec 2024-11-20T17:47:08,609 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732124828580-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:08,609 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732124828580-pool-0 {}] regionserver.HRegion(1722): Closing eaba082561ecbbfc53fef1900b914756, disabling compactions & flushes 2024-11-20T17:47:08,609 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732124828580-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:08,609 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732124828580-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:08,609 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732124828580-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. after waiting 0 ms 2024-11-20T17:47:08,609 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732124828580-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:08,609 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732124828580-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:08,609 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732124828580-pool-0 {}] regionserver.HRegion(1676): Region close journal for eaba082561ecbbfc53fef1900b914756: Waiting for close lock at 1732124828609Disabling compacts and flushes for region at 1732124828609Disabling writes for close at 1732124828609Writing region close event to WAL at 1732124828609Closed at 1732124828609 2024-11-20T17:47:08,610 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732124828580 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T17:47:08,611 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732124828610"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732124828610"}]},"ts":"1732124828610"} 2024-11-20T17:47:08,613 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T17:47:08,614 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732124828580 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T17:47:08,614 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732124828580","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124828614"}]},"ts":"1732124828614"} 2024-11-20T17:47:08,616 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732124828580, state=ENABLING in hbase:meta 2024-11-20T17:47:08,616 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {613581fcdc40=0} racks are {/default-rack=0} 2024-11-20T17:47:08,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T17:47:08,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T17:47:08,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:47:08,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:47:08,617 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:47:08,617 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:47:08,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T17:47:08,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN}] 2024-11-20T17:47:08,619 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN 2024-11-20T17:47:08,620 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN; state=OFFLINE, location=613581fcdc40,43195,1732124827163; forceNewPlan=false, retain=false 2024-11-20T17:47:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:08,771 INFO [613581fcdc40:37693 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T17:47:08,771 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eaba082561ecbbfc53fef1900b914756, regionState=OPENING, regionLocation=613581fcdc40,43195,1732124827163 2024-11-20T17:47:08,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN because future has completed 2024-11-20T17:47:08,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaba082561ecbbfc53fef1900b914756, server=613581fcdc40,43195,1732124827163}] 2024-11-20T17:47:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:08,936 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:08,936 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eaba082561ecbbfc53fef1900b914756, NAME => 'testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:08,936 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732124828580 eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,936 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:08,937 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,937 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,937 INFO [StoreOpener-eaba082561ecbbfc53fef1900b914756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,940 INFO [StoreOpener-eaba082561ecbbfc53fef1900b914756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaba082561ecbbfc53fef1900b914756 columnFamilyName family 2024-11-20T17:47:08,941 DEBUG [StoreOpener-eaba082561ecbbfc53fef1900b914756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:08,941 INFO [StoreOpener-eaba082561ecbbfc53fef1900b914756-1 {}] regionserver.HStore(327): Store=eaba082561ecbbfc53fef1900b914756/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:08,941 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,942 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,942 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,942 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,942 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,942 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,947 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:47:08,947 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eaba082561ecbbfc53fef1900b914756; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71854257, jitterRate=0.07071186602115631}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:47:08,948 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:08,948 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eaba082561ecbbfc53fef1900b914756: Running coprocessor pre-open hook at 1732124828937Writing region info on filesystem at 1732124828937Initializing all the Stores at 1732124828937Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124828937Cleaning up temporary data from old regions at 1732124828942 (+5 ms)Running coprocessor post-open hooks at 1732124828948 (+6 ms)Region opened successfully at 1732124828948 2024-11-20T17:47:08,949 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756., pid=6, masterSystemTime=1732124828933 2024-11-20T17:47:08,952 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:08,952 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:08,953 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eaba082561ecbbfc53fef1900b914756, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,43195,1732124827163 2024-11-20T17:47:08,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaba082561ecbbfc53fef1900b914756, server=613581fcdc40,43195,1732124827163 because future has completed 2024-11-20T17:47:08,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T17:47:08,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eaba082561ecbbfc53fef1900b914756, server=613581fcdc40,43195,1732124827163 in 179 msec 2024-11-20T17:47:08,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T17:47:08,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN in 343 msec 2024-11-20T17:47:08,963 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732124828580 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T17:47:08,963 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732124828580","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124828963"}]},"ts":"1732124828963"} 2024-11-20T17:47:08,966 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732124828580, state=ENABLED in hbase:meta 2024-11-20T17:47:08,967 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732124828580 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T17:47:08,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732124828580 in 382 msec 2024-11-20T17:47:09,044 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T17:47:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:09,214 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732124828580 completed 2024-11-20T17:47:09,214 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732124828580 get assigned. Timeout = 60000ms 2024-11-20T17:47:09,215 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:47:09,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732124828580 assigned to meta. Checking AM states. 2024-11-20T17:47:09,223 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:47:09,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732124828580 assigned. 2024-11-20T17:47:09,229 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732124828580', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756., hostname=613581fcdc40,43195,1732124827163, seqNum=2] 2024-11-20T17:47:09,235 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eaba082561ecbbfc53fef1900b914756 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T17:47:09,244 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 410e221a30844effaa9b32c203213597 2024-11-20T17:47:09,257 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/.tmp/family/410e221a30844effaa9b32c203213597 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T17:47:09,258 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/.tmp/family/410e221a30844effaa9b32c203213597 2024-11-20T17:47:09,260 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/.tmp/family/410e221a30844effaa9b32c203213597 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597 2024-11-20T17:47:09,262 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597, delay=977 ms 2024-11-20T17:47:09,263 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T17:47:09,263 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for eaba082561ecbbfc53fef1900b914756 in 28ms, sequenceid=6, compaction requested=false 2024-11-20T17:47:09,263 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732124828580' 2024-11-20T17:47:09,264 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for eaba082561ecbbfc53fef1900b914756: 2024-11-20T17:47:10,007 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T17:47:10,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T17:47:10,012 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T17:47:10,012 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T17:47:10,013 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T17:47:10,013 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T17:47:10,014 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732124828580 2024-11-20T17:47:10,014 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732124828580 Metrics about Tables on a single HBase RegionServer 2024-11-20T17:47:10,240 DEBUG [hfile-prefetch-1732124826286 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 410e221a30844effaa9b32c203213597_0. 2024-11-20T17:47:10,241 DEBUG [hfile-prefetch-1732124826286 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 410e221a30844effaa9b32c203213597_117. 2024-11-20T17:47:10,241 DEBUG [hfile-prefetch-1732124826286 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597, with total blocks 2, and data blocks 1 2024-11-20T17:47:10,241 DEBUG [hfile-prefetch-1732124826286 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T17:47:10,241 DEBUG [hfile-prefetch-1732124826286 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T17:47:10,241 DEBUG [hfile-prefetch-1732124826286 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597. Will sleep 100ms and try the verification again. 2024-11-20T17:47:10,264 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;613581fcdc40:43195,5,FailOnTimeoutGroup] 2024-11-20T17:47:10,265 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '613581fcdc40,43195,1732124827163' ***** 2024-11-20T17:47:10,265 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-20T17:47:10,265 INFO [RS:0;613581fcdc40:43195 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T17:47:10,331 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T17:47:10,332 INFO [regionserver/613581fcdc40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:10,338 INFO [RS:0;613581fcdc40:43195 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.persistence 2024-11-20T17:47:10,338 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T17:47:10,338 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T17:47:10,338 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T17:47:10,338 INFO [RS:0;613581fcdc40:43195 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T17:47:10,339 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T17:47:10,341 DEBUG [hfile-prefetch-1732124826286 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597, with total blocks 2, and data blocks 1 2024-11-20T17:47:10,342 DEBUG [hfile-prefetch-1732124826286 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T17:47:10,342 DEBUG [hfile-prefetch-1732124826286 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T17:47:10,342 INFO [hfile-prefetch-1732124826286 {}] bucket.BucketCache(2331): We found only 0 data blocks cached from a total of 1 for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-11-20T17:47:10,342 DEBUG [hfile-prefetch-1732124826286 {}] hfile.PrefetchExecutor(121): Prefetch completed for 410e221a30844effaa9b32c203213597 2024-11-20T17:47:10,348 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T17:47:10,348 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T17:47:10,348 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T17:47:10,348 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T17:47:10,349 DEBUG [RS:0;613581fcdc40:43195 {}] bucket.BucketCache(1487): Thread RS:0;613581fcdc40:43195 started persisting bucket cache to file 2024-11-20T17:47:10,349 DEBUG [RS:0;613581fcdc40:43195 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T17:47:10,349 DEBUG [RS:0;613581fcdc40:43195 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T17:47:10,352 DEBUG [RS:0;613581fcdc40:43195 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T17:47:10,352 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T17:47:10,352 DEBUG [RS:0;613581fcdc40:43195 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732124830349 2024-11-20T17:47:10,352 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732124830348 2024-11-20T17:47:10,352 DEBUG [RS:0;613581fcdc40:43195 {}] bucket.BucketCache(1509): Thread RS:0;613581fcdc40:43195 finished persisting bucket cache to file, renaming 2024-11-20T17:47:10,352 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T17:47:10,353 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T17:47:10,353 INFO [RS:0;613581fcdc40:43195 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T17:47:10,353 INFO [RS:0;613581fcdc40:43195 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T17:47:10,353 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(3091): Received CLOSE for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:10,353 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(959): stopping server 613581fcdc40,43195,1732124827163 2024-11-20T17:47:10,353 INFO [RS:0;613581fcdc40:43195 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:47:10,353 INFO [RS:0;613581fcdc40:43195 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;613581fcdc40:43195. 2024-11-20T17:47:10,353 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T17:47:10,353 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eaba082561ecbbfc53fef1900b914756, disabling compactions & flushes 2024-11-20T17:47:10,353 DEBUG [RS:0;613581fcdc40:43195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:47:10,353 DEBUG [RS:0;613581fcdc40:43195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:10,353 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:10,354 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:10,354 INFO [RS:0;613581fcdc40:43195 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T17:47:10,354 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. after waiting 0 ms 2024-11-20T17:47:10,354 INFO [RS:0;613581fcdc40:43195 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T17:47:10,354 INFO [RS:0;613581fcdc40:43195 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T17:47:10,354 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:10,354 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T17:47:10,354 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T17:47:10,354 DEBUG [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, eaba082561ecbbfc53fef1900b914756=testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756.} 2024-11-20T17:47:10,354 DEBUG [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:10,354 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T17:47:10,354 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:47:10,354 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:47:10,355 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:47:10,355 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:47:10,355 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:47:10,355 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-20T17:47:10,361 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T17:47:10,362 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:10,362 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eaba082561ecbbfc53fef1900b914756: Waiting for close lock at 1732124830353Running coprocessor pre-close hooks at 1732124830353Disabling compacts and flushes for region at 1732124830353Disabling writes for close at 1732124830354 (+1 ms)Writing region close event to WAL at 1732124830354Running coprocessor post-close hooks at 1732124830362 (+8 ms)Closed at 1732124830362 2024-11-20T17:47:10,362 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:10,370 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3d927bf546484d97a15bb0fec0f6b8bd 2024-11-20T17:47:10,382 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/info/3d927bf546484d97a15bb0fec0f6b8bd is 209, key is testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756./info:regioninfo/1732124828952/Put/seqid=0 2024-11-20T17:47:10,383 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/info/3d927bf546484d97a15bb0fec0f6b8bd 2024-11-20T17:47:10,391 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 012c3069c23b4a26a5640ba0ea28d1e0 2024-11-20T17:47:10,403 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/ns/012c3069c23b4a26a5640ba0ea28d1e0 is 43, key is default/ns:d/1732124828471/Put/seqid=0 2024-11-20T17:47:10,403 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/ns/012c3069c23b4a26a5640ba0ea28d1e0 2024-11-20T17:47:10,410 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e4a98912ea234592a64ecab120f3aa6b 2024-11-20T17:47:10,422 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/table/e4a98912ea234592a64ecab120f3aa6b is 80, key is testBlockEvictionOnGracefulStop1732124828580/table:state/1732124828963/Put/seqid=0 2024-11-20T17:47:10,422 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/table/e4a98912ea234592a64ecab120f3aa6b 2024-11-20T17:47:10,425 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/info/3d927bf546484d97a15bb0fec0f6b8bd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/3d927bf546484d97a15bb0fec0f6b8bd 2024-11-20T17:47:10,426 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/3d927bf546484d97a15bb0fec0f6b8bd, delay=946 ms 2024-11-20T17:47:10,427 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/3d927bf546484d97a15bb0fec0f6b8bd, entries=10, sequenceid=11, filesize=7.0 K 2024-11-20T17:47:10,427 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/ns/012c3069c23b4a26a5640ba0ea28d1e0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/012c3069c23b4a26a5640ba0ea28d1e0 2024-11-20T17:47:10,428 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/012c3069c23b4a26a5640ba0ea28d1e0, delay=924 ms 2024-11-20T17:47:10,429 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/012c3069c23b4a26a5640ba0ea28d1e0, entries=2, sequenceid=11, filesize=5.0 K 2024-11-20T17:47:10,429 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/table/e4a98912ea234592a64ecab120f3aa6b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/e4a98912ea234592a64ecab120f3aa6b 2024-11-20T17:47:10,430 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/e4a98912ea234592a64ecab120f3aa6b, delay=941 ms 2024-11-20T17:47:10,431 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/e4a98912ea234592a64ecab120f3aa6b, entries=2, sequenceid=11, filesize=5.3 K 2024-11-20T17:47:10,432 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 77ms, sequenceid=11, compaction requested=false 2024-11-20T17:47:10,432 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/3d927bf546484d97a15bb0fec0f6b8bd 2024-11-20T17:47:10,433 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/012c3069c23b4a26a5640ba0ea28d1e0 2024-11-20T17:47:10,433 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/e4a98912ea234592a64ecab120f3aa6b 2024-11-20T17:47:10,437 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-20T17:47:10,437 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T17:47:10,437 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:47:10,438 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732124830354Running coprocessor pre-close hooks at 1732124830354Disabling compacts and flushes for region at 1732124830354Disabling writes for close at 1732124830355 (+1 ms)Obtaining lock to block concurrent updates at 1732124830355Preparing flush snapshotting stores in 1588230740 at 1732124830355Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732124830355Flushing stores of hbase:meta,,1.1588230740 at 1732124830356 (+1 ms)Flushing 1588230740/info: creating writer at 1732124830356Flushing 1588230740/info: appending metadata at 1732124830382 (+26 ms)Flushing 1588230740/info: closing flushed file at 1732124830382Flushing 1588230740/ns: creating writer at 1732124830385 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732124830402 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732124830402Flushing 1588230740/table: creating writer at 1732124830405 (+3 ms)Flushing 1588230740/table: appending metadata at 1732124830421 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732124830421Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4584a97d: reopening flushed file at 1732124830424 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26b4b9fb: reopening flushed file at 1732124830427 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d8f44c8: reopening flushed file at 1732124830429 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 77ms, sequenceid=11, compaction requested=false at 1732124830432 (+3 ms)Writing region close event to WAL at 1732124830434 (+2 ms)Running coprocessor post-close hooks at 1732124830437 (+3 ms)Closed at 1732124830437 2024-11-20T17:47:10,438 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T17:47:10,555 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(976): stopping server 613581fcdc40,43195,1732124827163; all regions closed. 2024-11-20T17:47:10,560 DEBUG [RS:0;613581fcdc40:43195 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs 2024-11-20T17:47:10,560 INFO [RS:0;613581fcdc40:43195 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C43195%2C1732124827163.meta:.meta(num 1732124828374) 2024-11-20T17:47:10,564 DEBUG [RS:0;613581fcdc40:43195 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs 2024-11-20T17:47:10,564 INFO [RS:0;613581fcdc40:43195 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C43195%2C1732124827163:(num 1732124828065) 2024-11-20T17:47:10,564 DEBUG [RS:0;613581fcdc40:43195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:10,564 INFO [RS:0;613581fcdc40:43195 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:10,564 INFO [RS:0;613581fcdc40:43195 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:47:10,564 INFO [RS:0;613581fcdc40:43195 {}] hbase.ChoreService(370): Chore service for: regionserver/613581fcdc40:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T17:47:10,565 INFO [RS:0;613581fcdc40:43195 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:47:10,565 INFO [regionserver/613581fcdc40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:47:10,565 INFO [RS:0;613581fcdc40:43195 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43195 2024-11-20T17:47:10,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:47:10,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/613581fcdc40,43195,1732124827163 2024-11-20T17:47:10,588 INFO [RS:0;613581fcdc40:43195 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:47:10,599 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [613581fcdc40,43195,1732124827163] 2024-11-20T17:47:10,609 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/613581fcdc40,43195,1732124827163 already deleted, retry=false 2024-11-20T17:47:10,609 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 613581fcdc40,43195,1732124827163 on 613581fcdc40,37693,1732124826990 2024-11-20T17:47:10,615 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 613581fcdc40,43195,1732124827163, splitWal=true, meta=true 2024-11-20T17:47:10,617 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 613581fcdc40,43195,1732124827163, splitWal=true, meta=true 2024-11-20T17:47:10,617 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 613581fcdc40,43195,1732124827163 (carryingMeta=true) 613581fcdc40,43195,1732124827163/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@17af07e[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T17:47:10,617 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 613581fcdc40,43195,1732124827163, splitWal=true, meta=true, isMeta: true 2024-11-20T17:47:10,618 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163-splitting 2024-11-20T17:47:10,618 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163-splitting dir is empty, no logs to split. 2024-11-20T17:47:10,618 INFO [PEWorker-2 {}] master.SplitWALManager(105): 613581fcdc40,43195,1732124827163 WAL count=0, meta=true 2024-11-20T17:47:10,620 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163-splitting dir is empty, no logs to split. 2024-11-20T17:47:10,620 INFO [PEWorker-2 {}] master.SplitWALManager(105): 613581fcdc40,43195,1732124827163 WAL count=0, meta=true 2024-11-20T17:47:10,620 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 613581fcdc40,43195,1732124827163 WAL splitting is done? wals=0, meta=true 2024-11-20T17:47:10,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T17:47:10,622 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:47:10,623 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:47:10,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:10,699 INFO [RS:0;613581fcdc40:43195 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:47:10,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43195-0x101595b58fd0001, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:10,699 INFO [RS:0;613581fcdc40:43195 {}] regionserver.HRegionServer(1031): Exiting; stopping=613581fcdc40,43195,1732124827163; zookeeper connection closed. 2024-11-20T17:47:10,699 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@57bdbb6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@57bdbb6 2024-11-20T17:47:10,773 DEBUG [613581fcdc40:37693 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T17:47:10,774 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=613581fcdc40,45773,1732124827218 2024-11-20T17:47:10,775 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,45773,1732124827218, state=OPENING 2024-11-20T17:47:10,786 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/613581fcdc40:0 server-side Connection retries=45 2024-11-20T17:47:10,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:10,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:10,787 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:47:10,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:10,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:47:10,787 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:47:10,787 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:47:10,788 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41829 2024-11-20T17:47:10,790 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41829 connecting to ZooKeeper ensemble=127.0.0.1:56636 2024-11-20T17:47:10,791 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:10,791 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:10,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:10,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:10,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:10,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:10,798 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:47:10,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,45773,1732124827218}] 2024-11-20T17:47:10,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:418290x0, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:47:10,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41829-0x101595b58fd0003 connected 2024-11-20T17:47:10,896 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-20T17:47:10,896 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-20T17:47:10,898 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:47:10,899 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.cache 2024-11-20T17:47:10,900 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T17:47:10,900 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T17:47:10,900 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T17:47:10,901 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T17:47:10,901 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T17:47:10,903 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-20T17:47:10,904 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-20T17:47:10,905 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:47:10,906 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T17:47:10,908 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:47:10,908 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-20T17:47:10,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41829 2024-11-20T17:47:10,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41829 2024-11-20T17:47:10,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41829 2024-11-20T17:47:10,909 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T17:47:10,909 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-20T17:47:10,909 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.persistence is complete. 2024-11-20T17:47:10,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41829 2024-11-20T17:47:10,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41829 2024-11-20T17:47:10,913 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(746): ClusterId : 57186c09-30f7-4530-a0c7-f0ca75a2036f 2024-11-20T17:47:10,913 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:47:10,947 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:47:10,947 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:47:10,956 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T17:47:10,957 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:47:10,958 DEBUG [RS:2;613581fcdc40:41829 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb5ec88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:47:10,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46723, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T17:47:10,960 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T17:47:10,960 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:10,961 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T17:47:10,962 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=613581fcdc40%2C45773%2C1732124827218.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,45773,1732124827218, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs, maxLogs=55 2024-11-20T17:47:10,976 DEBUG [RS:2;613581fcdc40:41829 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;613581fcdc40:41829 2024-11-20T17:47:10,976 INFO [RS:2;613581fcdc40:41829 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:47:10,976 INFO [RS:2;613581fcdc40:41829 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:47:10,976 DEBUG [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:47:10,977 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(2659): reportForDuty to master=613581fcdc40,37693,1732124826990 with port=41829, startcode=1732124830786 2024-11-20T17:47:10,977 DEBUG [RS:2;613581fcdc40:41829 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:47:10,980 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59439, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:47:10,980 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37693 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 613581fcdc40,41829,1732124830786 2024-11-20T17:47:10,980 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37693 {}] master.ServerManager(517): Registering regionserver=613581fcdc40,41829,1732124830786 2024-11-20T17:47:10,982 DEBUG [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec 2024-11-20T17:47:10,983 DEBUG [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T17:47:10,983 DEBUG [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:47:10,992 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,45773,1732124827218/613581fcdc40%2C45773%2C1732124827218.meta.1732124830963.meta 2024-11-20T17:47:10,993 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:10,993 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:10,994 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T17:47:10,994 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T17:47:10,994 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T17:47:10,994 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T17:47:10,994 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:10,995 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T17:47:10,995 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T17:47:10,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T17:47:10,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:47:10,997 DEBUG [RS:2;613581fcdc40:41829 {}] zookeeper.ZKUtil(111): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/613581fcdc40,41829,1732124830786 2024-11-20T17:47:10,997 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:47:10,997 WARN [RS:2;613581fcdc40:41829 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:47:10,997 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:10,997 INFO [RS:2;613581fcdc40:41829 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:10,997 DEBUG [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,41829,1732124830786 2024-11-20T17:47:10,998 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [613581fcdc40,41829,1732124830786] 2024-11-20T17:47:10,999 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/3d927bf546484d97a15bb0fec0f6b8bd, delay=918 ms 2024-11-20T17:47:11,001 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/3d927bf546484d97a15bb0fec0f6b8bd 2024-11-20T17:47:11,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:11,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T17:47:11,003 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:47:11,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:11,004 INFO [RS:2;613581fcdc40:41829 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:47:11,005 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/012c3069c23b4a26a5640ba0ea28d1e0, delay=990 ms 2024-11-20T17:47:11,007 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/012c3069c23b4a26a5640ba0ea28d1e0 2024-11-20T17:47:11,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:11,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T17:47:11,008 INFO [RS:2;613581fcdc40:41829 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:47:11,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:47:11,008 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:11,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:11,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T17:47:11,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:47:11,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:11,012 INFO [RS:2;613581fcdc40:41829 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:47:11,012 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,013 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:47:11,013 INFO [RS:2;613581fcdc40:41829 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:47:11,013 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,013 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/e4a98912ea234592a64ecab120f3aa6b, delay=981 ms 2024-11-20T17:47:11,014 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,014 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,014 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,014 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,014 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,014 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:47:11,014 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,014 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,014 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,015 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,015 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,015 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/e4a98912ea234592a64ecab120f3aa6b 2024-11-20T17:47:11,015 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:11,015 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:11,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:11,015 DEBUG [RS:2;613581fcdc40:41829 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:11,015 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:47:11,016 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,016 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,016 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,016 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,016 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:11,016 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,016 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,41829,1732124830786-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:47:11,017 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740 2024-11-20T17:47:11,017 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:47:11,017 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:47:11,018 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:47:11,019 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:47:11,020 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59433396, jitterRate=-0.11437338590621948}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:47:11,020 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T17:47:11,021 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732124830995Writing region info on filesystem at 1732124830995Initializing all the Stores at 1732124830995Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124830995Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124830996 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124830996Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124830996Cleaning up temporary data from old regions at 1732124831017 (+21 ms)Running coprocessor post-open hooks at 1732124831020 (+3 ms)Region opened successfully at 1732124831021 (+1 ms) 2024-11-20T17:47:11,022 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732124830956 2024-11-20T17:47:11,027 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T17:47:11,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=613581fcdc40,45773,1732124827218 2024-11-20T17:47:11,027 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T17:47:11,029 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,45773,1732124827218, state=OPEN 2024-11-20T17:47:11,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:11,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:11,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:11,041 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=613581fcdc40,45773,1732124827218 2024-11-20T17:47:11,041 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:11,041 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:11,041 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:11,042 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:47:11,043 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,41829,1732124830786-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,043 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,043 INFO [RS:2;613581fcdc40:41829 {}] regionserver.Replication(171): 613581fcdc40,41829,1732124830786 started 2024-11-20T17:47:11,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T17:47:11,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,45773,1732124827218 in 243 msec 2024-11-20T17:47:11,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T17:47:11,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 424 msec 2024-11-20T17:47:11,050 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 613581fcdc40,43195,1732124827163 had 2 regions 2024-11-20T17:47:11,051 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 613581fcdc40,43195,1732124827163, splitWal=true, meta=true, isMeta: false 2024-11-20T17:47:11,052 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163-splitting dir is empty, no logs to split. 2024-11-20T17:47:11,052 INFO [PEWorker-2 {}] master.SplitWALManager(105): 613581fcdc40,43195,1732124827163 WAL count=0, meta=false 2024-11-20T17:47:11,055 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,43195,1732124827163-splitting dir is empty, no logs to split. 2024-11-20T17:47:11,055 INFO [PEWorker-2 {}] master.SplitWALManager(105): 613581fcdc40,43195,1732124827163 WAL count=0, meta=false 2024-11-20T17:47:11,055 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 613581fcdc40,43195,1732124827163 WAL splitting is done? wals=0, meta=false 2024-11-20T17:47:11,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN}] 2024-11-20T17:47:11,058 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN 2024-11-20T17:47:11,059 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:47:11,067 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:11,067 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(1482): Serving as 613581fcdc40,41829,1732124830786, RpcServer on 613581fcdc40/172.17.0.2:41829, sessionid=0x101595b58fd0003 2024-11-20T17:47:11,067 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:47:11,067 DEBUG [RS:2;613581fcdc40:41829 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 613581fcdc40,41829,1732124830786 2024-11-20T17:47:11,067 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,41829,1732124830786' 2024-11-20T17:47:11,068 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:47:11,068 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:47:11,069 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:47:11,069 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:47:11,069 DEBUG [RS:2;613581fcdc40:41829 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 613581fcdc40,41829,1732124830786 2024-11-20T17:47:11,069 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,41829,1732124830786' 2024-11-20T17:47:11,069 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:47:11,070 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:47:11,070 DEBUG [RS:2;613581fcdc40:41829 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:47:11,071 INFO [RS:2;613581fcdc40:41829 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:47:11,071 INFO [RS:2;613581fcdc40:41829 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:47:11,172 INFO [RS:2;613581fcdc40:41829 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:47:11,173 INFO [RS:2;613581fcdc40:41829 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=613581fcdc40%2C41829%2C1732124830786, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,41829,1732124830786, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs, maxLogs=55 2024-11-20T17:47:11,199 INFO [RS:2;613581fcdc40:41829 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/WALs/613581fcdc40,41829,1732124830786/613581fcdc40%2C41829%2C1732124830786.1732124831174 2024-11-20T17:47:11,200 DEBUG [RS:2;613581fcdc40:41829 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T17:47:11,209 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(204): Hosts are {613581fcdc40=0} racks are {/default-rack=0} 2024-11-20T17:47:11,210 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T17:47:11,210 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T17:47:11,210 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:47:11,210 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:47:11,210 INFO [613581fcdc40:37693 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:47:11,210 INFO [613581fcdc40:37693 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:47:11,210 DEBUG [613581fcdc40:37693 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T17:47:11,210 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=eaba082561ecbbfc53fef1900b914756, regionState=OPENING, regionLocation=613581fcdc40,41829,1732124830786 2024-11-20T17:47:11,212 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 613581fcdc40:43195 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 613581fcdc40/172.17.0.2:43195 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:11,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=613581fcdc40,43195,1732124827163, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=613581fcdc40,43195,1732124827163, seqNum=-1, error=java.net.ConnectException: Call to address=613581fcdc40:43195 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 613581fcdc40/172.17.0.2:43195 2024-11-20T17:47:11,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=613581fcdc40,43195,1732124827163, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T17:47:11,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=613581fcdc40,43195,1732124827163, seqNum=-1 from cache 2024-11-20T17:47:11,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 613581fcdc40:43195 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 613581fcdc40/172.17.0.2:43195 2024-11-20T17:47:11,321 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:11,322 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,45773,1732124827218, seqNum=-1] 2024-11-20T17:47:11,322 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:11,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50965, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:11,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN because future has completed 2024-11-20T17:47:11,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaba082561ecbbfc53fef1900b914756, server=613581fcdc40,41829,1732124830786}] 2024-11-20T17:47:11,482 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T17:47:11,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40709, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T17:47:11,487 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:11,487 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => eaba082561ecbbfc53fef1900b914756, NAME => 'testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:11,488 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732124828580 eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,488 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:11,488 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,488 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,490 INFO [StoreOpener-eaba082561ecbbfc53fef1900b914756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,491 INFO [StoreOpener-eaba082561ecbbfc53fef1900b914756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaba082561ecbbfc53fef1900b914756 columnFamilyName family 2024-11-20T17:47:11,491 DEBUG [StoreOpener-eaba082561ecbbfc53fef1900b914756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:11,493 DEBUG [StoreFileOpener-eaba082561ecbbfc53fef1900b914756-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597, delay=946 ms 2024-11-20T17:47:11,494 DEBUG [StoreOpener-eaba082561ecbbfc53fef1900b914756-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597 2024-11-20T17:47:11,495 INFO [StoreOpener-eaba082561ecbbfc53fef1900b914756-1 {}] regionserver.HStore(327): Store=eaba082561ecbbfc53fef1900b914756/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:11,495 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,496 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,496 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,497 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,497 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,498 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,499 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened eaba082561ecbbfc53fef1900b914756; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63899547, jitterRate=-0.04782254993915558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:47:11,499 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,500 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for eaba082561ecbbfc53fef1900b914756: Running coprocessor pre-open hook at 1732124831489Writing region info on filesystem at 1732124831489Initializing all the Stores at 1732124831489Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124831489Cleaning up temporary data from old regions at 1732124831497 (+8 ms)Running coprocessor post-open hooks at 1732124831499 (+2 ms)Region opened successfully at 1732124831500 (+1 ms) 2024-11-20T17:47:11,501 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756., pid=11, masterSystemTime=1732124831481 2024-11-20T17:47:11,504 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:11,505 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:11,505 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=eaba082561ecbbfc53fef1900b914756, regionState=OPEN, openSeqNum=10, regionLocation=613581fcdc40,41829,1732124830786 2024-11-20T17:47:11,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaba082561ecbbfc53fef1900b914756, server=613581fcdc40,41829,1732124830786 because future has completed 2024-11-20T17:47:11,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T17:47:11,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure eaba082561ecbbfc53fef1900b914756, server=613581fcdc40,41829,1732124830786 in 181 msec 2024-11-20T17:47:11,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-20T17:47:11,515 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 613581fcdc40,43195,1732124827163 after splitting done 2024-11-20T17:47:11,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732124828580, region=eaba082561ecbbfc53fef1900b914756, ASSIGN in 456 msec 2024-11-20T17:47:11,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 613581fcdc40,43195,1732124827163, splitWal=true, meta=true in 904 msec 2024-11-20T17:47:11,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T17:47:11,568 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:47:11,568 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:47:11,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:11,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:11,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:47:11,568 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T17:47:11,569 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1264405596, stopped=false 2024-11-20T17:47:11,569 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=613581fcdc40,37693,1732124826990 2024-11-20T17:47:11,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:11,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:11,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:11,620 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:47:11,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:11,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:11,620 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:47:11,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:11,621 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:47:11,621 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:11,621 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:11,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:11,621 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '613581fcdc40,45773,1732124827218' ***** 2024-11-20T17:47:11,621 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:11,621 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T17:47:11,621 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '613581fcdc40,41829,1732124830786' ***** 2024-11-20T17:47:11,621 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T17:47:11,621 INFO [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T17:47:11,621 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T17:47:11,621 INFO [RS:2;613581fcdc40:41829 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T17:47:11,622 INFO [RS:2;613581fcdc40:41829 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.persistence 2024-11-20T17:47:11,622 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T17:47:11,622 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T17:47:11,622 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T17:47:11,622 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T17:47:11,622 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T17:47:11,622 INFO [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da3f211b-0de6-c99a-464d-dee04fb551a1/bucket.persistence 2024-11-20T17:47:11,622 INFO [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T17:47:11,622 INFO [RS:2;613581fcdc40:41829 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T17:47:11,622 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T17:47:11,622 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T17:47:11,631 INFO [regionserver/613581fcdc40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:11,631 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T17:47:11,632 DEBUG [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1487): Thread RS:1;613581fcdc40:45773 started persisting bucket cache to file 2024-11-20T17:47:11,632 DEBUG [RS:2;613581fcdc40:41829 {}] bucket.BucketCache(1487): Thread RS:2;613581fcdc40:41829 started persisting bucket cache to file 2024-11-20T17:47:11,632 DEBUG [RS:2;613581fcdc40:41829 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T17:47:11,632 DEBUG [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T17:47:11,632 DEBUG [RS:2;613581fcdc40:41829 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T17:47:11,632 DEBUG [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T17:47:11,632 INFO [regionserver/613581fcdc40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:11,639 DEBUG [RS:2;613581fcdc40:41829 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T17:47:11,639 DEBUG [RS:2;613581fcdc40:41829 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732124831632 2024-11-20T17:47:11,639 DEBUG [RS:2;613581fcdc40:41829 {}] bucket.BucketCache(1509): Thread RS:2;613581fcdc40:41829 finished persisting bucket cache to file, renaming 2024-11-20T17:47:11,639 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T17:47:11,639 INFO [RS:2;613581fcdc40:41829 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T17:47:11,639 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T17:47:11,639 INFO [RS:2;613581fcdc40:41829 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T17:47:11,640 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(3091): Received CLOSE for eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,640 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(959): stopping server 613581fcdc40,41829,1732124830786 2024-11-20T17:47:11,640 INFO [RS:2;613581fcdc40:41829 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:47:11,641 INFO [RS:2;613581fcdc40:41829 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;613581fcdc40:41829. 2024-11-20T17:47:11,641 DEBUG [RS:2;613581fcdc40:41829 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:47:11,641 DEBUG [RS:2;613581fcdc40:41829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:11,641 ERROR [RS:1;613581fcdc40:45773 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:11,641 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T17:47:11,641 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T17:47:11,641 DEBUG [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(1325): Online Regions={eaba082561ecbbfc53fef1900b914756=testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756.} 2024-11-20T17:47:11,641 DEBUG [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T17:47:11,641 DEBUG [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(1351): Waiting on eaba082561ecbbfc53fef1900b914756 2024-11-20T17:47:11,641 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eaba082561ecbbfc53fef1900b914756, disabling compactions & flushes 2024-11-20T17:47:11,641 DEBUG [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732124831632 2024-11-20T17:47:11,641 DEBUG [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1509): Thread RS:1;613581fcdc40:45773 finished persisting bucket cache to file, renaming 2024-11-20T17:47:11,641 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:11,641 WARN [RS:1;613581fcdc40:45773 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-20T17:47:11,641 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:11,641 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T17:47:11,641 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. after waiting 0 ms 2024-11-20T17:47:11,642 INFO [RS:1;613581fcdc40:45773 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T17:47:11,642 INFO [RS:1;613581fcdc40:45773 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T17:47:11,642 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(959): stopping server 613581fcdc40,45773,1732124827218 2024-11-20T17:47:11,642 INFO [RS:1;613581fcdc40:45773 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:47:11,640 WARN [Thread-217 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-20T17:47:11,642 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:11,642 INFO [RS:1;613581fcdc40:45773 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;613581fcdc40:45773. 2024-11-20T17:47:11,642 DEBUG [RS:1;613581fcdc40:45773 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:47:11,642 DEBUG [RS:1;613581fcdc40:45773 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:11,642 INFO [RS:1;613581fcdc40:45773 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T17:47:11,642 INFO [RS:1;613581fcdc40:45773 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T17:47:11,642 INFO [RS:1;613581fcdc40:45773 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T17:47:11,642 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T17:47:11,643 DEBUG [StoreFileCloser-eaba082561ecbbfc53fef1900b914756-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/family/410e221a30844effaa9b32c203213597 2024-11-20T17:47:11,644 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T17:47:11,644 DEBUG [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T17:47:11,644 DEBUG [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:47:11,644 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T17:47:11,644 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:47:11,645 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:47:11,645 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:47:11,645 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:47:11,645 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:47:11,645 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-20T17:47:11,652 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/default/testBlockEvictionOnGracefulStop1732124828580/eaba082561ecbbfc53fef1900b914756/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T17:47:11,652 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:11,652 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eaba082561ecbbfc53fef1900b914756: Waiting for close lock at 1732124831641Running coprocessor pre-close hooks at 1732124831641Disabling compacts and flushes for region at 1732124831641Disabling writes for close at 1732124831641Writing region close event to WAL at 1732124831643 (+2 ms)Running coprocessor post-close hooks at 1732124831652 (+9 ms)Closed at 1732124831652 2024-11-20T17:47:11,652 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756. 2024-11-20T17:47:11,657 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 32a8225e7da5489f8e0f444f7db55127 2024-11-20T17:47:11,672 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/info/32a8225e7da5489f8e0f444f7db55127 is 209, key is testBlockEvictionOnGracefulStop1732124828580,,1732124828584.eaba082561ecbbfc53fef1900b914756./info:regioninfo/1732124831505/Put/seqid=0 2024-11-20T17:47:11,673 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/info/32a8225e7da5489f8e0f444f7db55127 2024-11-20T17:47:11,676 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/.tmp/info/32a8225e7da5489f8e0f444f7db55127 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/32a8225e7da5489f8e0f444f7db55127 2024-11-20T17:47:11,677 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/32a8225e7da5489f8e0f444f7db55127, delay=912 ms 2024-11-20T17:47:11,678 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/32a8225e7da5489f8e0f444f7db55127, entries=8, sequenceid=19, filesize=6.6 K 2024-11-20T17:47:11,679 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=19, compaction requested=false 2024-11-20T17:47:11,680 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/3d927bf546484d97a15bb0fec0f6b8bd 2024-11-20T17:47:11,680 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/info/32a8225e7da5489f8e0f444f7db55127 2024-11-20T17:47:11,681 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/ns/012c3069c23b4a26a5640ba0ea28d1e0 2024-11-20T17:47:11,684 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/table/e4a98912ea234592a64ecab120f3aa6b 2024-11-20T17:47:11,689 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-20T17:47:11,690 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T17:47:11,690 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:47:11,690 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732124831644Running coprocessor pre-close hooks at 1732124831644Disabling compacts and flushes for region at 1732124831644Disabling writes for close at 1732124831645 (+1 ms)Obtaining lock to block concurrent updates at 1732124831645Preparing flush snapshotting stores in 1588230740 at 1732124831645Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732124831646 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732124831646Flushing 1588230740/info: creating writer at 1732124831647 (+1 ms)Flushing 1588230740/info: appending metadata at 1732124831672 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732124831672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@453440af: reopening flushed file at 1732124831675 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=19, compaction requested=false at 1732124831679 (+4 ms)Writing region close event to WAL at 1732124831685 (+6 ms)Running coprocessor post-close hooks at 1732124831689 (+4 ms)Closed at 1732124831690 (+1 ms) 2024-11-20T17:47:11,690 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T17:47:11,841 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(976): stopping server 613581fcdc40,41829,1732124830786; all regions closed. 2024-11-20T17:47:11,843 DEBUG [RS:2;613581fcdc40:41829 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs 2024-11-20T17:47:11,843 INFO [RS:2;613581fcdc40:41829 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C41829%2C1732124830786:(num 1732124831174) 2024-11-20T17:47:11,843 DEBUG [RS:2;613581fcdc40:41829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:11,843 INFO [RS:2;613581fcdc40:41829 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:11,844 INFO [RS:2;613581fcdc40:41829 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:47:11,844 INFO [RS:2;613581fcdc40:41829 {}] hbase.ChoreService(370): Chore service for: regionserver/613581fcdc40:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T17:47:11,844 INFO [RS:2;613581fcdc40:41829 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T17:47:11,844 INFO [regionserver/613581fcdc40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:47:11,844 INFO [RS:2;613581fcdc40:41829 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T17:47:11,844 INFO [RS:2;613581fcdc40:41829 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T17:47:11,844 INFO [RS:2;613581fcdc40:41829 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:47:11,844 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(976): stopping server 613581fcdc40,45773,1732124827218; all regions closed. 2024-11-20T17:47:11,844 INFO [RS:2;613581fcdc40:41829 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41829 2024-11-20T17:47:11,846 DEBUG [RS:1;613581fcdc40:45773 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs 2024-11-20T17:47:11,846 INFO [RS:1;613581fcdc40:45773 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C45773%2C1732124827218.meta:.meta(num 1732124830963) 2024-11-20T17:47:11,849 DEBUG [RS:1;613581fcdc40:45773 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/oldWALs 2024-11-20T17:47:11,849 INFO [RS:1;613581fcdc40:45773 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C45773%2C1732124827218:(num 1732124828067) 2024-11-20T17:47:11,849 DEBUG [RS:1;613581fcdc40:45773 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:11,849 INFO [RS:1;613581fcdc40:45773 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:47:11,849 INFO [RS:1;613581fcdc40:45773 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:47:11,849 INFO [RS:1;613581fcdc40:45773 {}] hbase.ChoreService(370): Chore service for: regionserver/613581fcdc40:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T17:47:11,849 INFO [RS:1;613581fcdc40:45773 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:47:11,849 INFO [regionserver/613581fcdc40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:47:11,850 INFO [RS:1;613581fcdc40:45773 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45773 2024-11-20T17:47:11,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:47:11,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/613581fcdc40,41829,1732124830786 2024-11-20T17:47:11,861 INFO [RS:2;613581fcdc40:41829 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:47:11,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/613581fcdc40,45773,1732124827218 2024-11-20T17:47:11,872 INFO [RS:1;613581fcdc40:45773 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:47:11,883 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [613581fcdc40,45773,1732124827218] 2024-11-20T17:47:11,903 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/613581fcdc40,45773,1732124827218 already deleted, retry=false 2024-11-20T17:47:11,903 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 613581fcdc40,45773,1732124827218 expired; onlineServers=1 2024-11-20T17:47:11,904 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [613581fcdc40,41829,1732124830786] 2024-11-20T17:47:11,914 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/613581fcdc40,41829,1732124830786 already deleted, retry=false 2024-11-20T17:47:11,914 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 613581fcdc40,41829,1732124830786 expired; onlineServers=0 2024-11-20T17:47:11,914 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '613581fcdc40,37693,1732124826990' ***** 2024-11-20T17:47:11,914 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T17:47:11,914 INFO [M:0;613581fcdc40:37693 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:47:11,914 INFO [M:0;613581fcdc40:37693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:47:11,915 DEBUG [M:0;613581fcdc40:37693 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T17:47:11,915 DEBUG [M:0;613581fcdc40:37693 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T17:47:11,915 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T17:47:11,915 DEBUG [master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732124827705 {}] cleaner.HFileCleaner(306): Exit Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732124827705,5,FailOnTimeoutGroup] 2024-11-20T17:47:11,915 DEBUG [master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732124827705 {}] cleaner.HFileCleaner(306): Exit Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732124827705,5,FailOnTimeoutGroup] 2024-11-20T17:47:11,915 INFO [M:0;613581fcdc40:37693 {}] hbase.ChoreService(370): Chore service for: master/613581fcdc40:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T17:47:11,915 INFO [M:0;613581fcdc40:37693 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:47:11,915 DEBUG [M:0;613581fcdc40:37693 {}] master.HMaster(1795): Stopping service threads 2024-11-20T17:47:11,915 INFO [M:0;613581fcdc40:37693 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T17:47:11,915 INFO [M:0;613581fcdc40:37693 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:47:11,916 INFO [M:0;613581fcdc40:37693 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:47:11,916 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T17:47:11,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T17:47:11,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:11,925 DEBUG [M:0;613581fcdc40:37693 {}] zookeeper.ZKUtil(347): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T17:47:11,925 WARN [M:0;613581fcdc40:37693 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T17:47:11,925 INFO [M:0;613581fcdc40:37693 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/.lastflushedseqids 2024-11-20T17:47:11,928 INFO [M:0;613581fcdc40:37693 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:47:11,928 INFO [M:0;613581fcdc40:37693 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T17:47:11,928 DEBUG [M:0;613581fcdc40:37693 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T17:47:11,928 INFO [M:0;613581fcdc40:37693 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:11,929 DEBUG [M:0;613581fcdc40:37693 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:11,929 DEBUG [M:0;613581fcdc40:37693 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T17:47:11,929 DEBUG [M:0;613581fcdc40:37693 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:11,929 INFO [M:0;613581fcdc40:37693 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-20T17:47:11,938 DEBUG [M:0;613581fcdc40:37693 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on efbff4ab277548cabedfa714184cbe09 2024-11-20T17:47:11,951 DEBUG [M:0;613581fcdc40:37693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efbff4ab277548cabedfa714184cbe09 is 82, key is hbase:meta,,1/info:regioninfo/1732124831027/Put/seqid=0 2024-11-20T17:47:11,952 INFO [M:0;613581fcdc40:37693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efbff4ab277548cabedfa714184cbe09 2024-11-20T17:47:11,959 DEBUG [M:0;613581fcdc40:37693 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 659629172ebe469885a1b53dd0e9c650 2024-11-20T17:47:11,979 DEBUG [M:0;613581fcdc40:37693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/659629172ebe469885a1b53dd0e9c650 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732124828969/Put/seqid=0 2024-11-20T17:47:11,980 INFO [M:0;613581fcdc40:37693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/659629172ebe469885a1b53dd0e9c650 2024-11-20T17:47:11,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:11,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41829-0x101595b58fd0003, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:11,983 INFO [RS:2;613581fcdc40:41829 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:47:11,983 INFO [RS:2;613581fcdc40:41829 {}] regionserver.HRegionServer(1031): Exiting; stopping=613581fcdc40,41829,1732124830786; zookeeper connection closed. 2024-11-20T17:47:11,983 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@64bac95f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@64bac95f 2024-11-20T17:47:11,988 DEBUG [M:0;613581fcdc40:37693 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f1c69fa33f6a4bab974d60c02750a9f4 2024-11-20T17:47:11,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:11,993 INFO [RS:1;613581fcdc40:45773 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:47:11,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45773-0x101595b58fd0002, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:11,993 INFO [RS:1;613581fcdc40:45773 {}] regionserver.HRegionServer(1031): Exiting; stopping=613581fcdc40,45773,1732124827218; zookeeper connection closed. 2024-11-20T17:47:11,994 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7be10eaa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7be10eaa 2024-11-20T17:47:11,994 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T17:47:12,003 DEBUG [M:0;613581fcdc40:37693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f1c69fa33f6a4bab974d60c02750a9f4 is 69, key is 613581fcdc40,41829,1732124830786/rs:state/1732124830981/Put/seqid=0 2024-11-20T17:47:12,004 INFO [M:0;613581fcdc40:37693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f1c69fa33f6a4bab974d60c02750a9f4 2024-11-20T17:47:12,005 INFO [M:0;613581fcdc40:37693 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f1c69fa33f6a4bab974d60c02750a9f4 2024-11-20T17:47:12,006 DEBUG [M:0;613581fcdc40:37693 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efbff4ab277548cabedfa714184cbe09 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/efbff4ab277548cabedfa714184cbe09 2024-11-20T17:47:12,008 INFO [M:0;613581fcdc40:37693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/efbff4ab277548cabedfa714184cbe09, entries=14, sequenceid=102, filesize=5.9 K 2024-11-20T17:47:12,008 DEBUG [M:0;613581fcdc40:37693 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/659629172ebe469885a1b53dd0e9c650 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/659629172ebe469885a1b53dd0e9c650 2024-11-20T17:47:12,010 INFO [M:0;613581fcdc40:37693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/659629172ebe469885a1b53dd0e9c650, entries=11, sequenceid=102, filesize=6.5 K 2024-11-20T17:47:12,011 DEBUG [M:0;613581fcdc40:37693 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f1c69fa33f6a4bab974d60c02750a9f4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f1c69fa33f6a4bab974d60c02750a9f4 2024-11-20T17:47:12,013 INFO [M:0;613581fcdc40:37693 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f1c69fa33f6a4bab974d60c02750a9f4 2024-11-20T17:47:12,013 INFO [M:0;613581fcdc40:37693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1cf15d0-f3d0-2f09-c3f3-cc2f8c585c05/b72993f5-377c-c55e-8511-a85cf03224ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f1c69fa33f6a4bab974d60c02750a9f4, entries=3, sequenceid=102, filesize=5.3 K 2024-11-20T17:47:12,014 INFO [M:0;613581fcdc40:37693 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=102, compaction requested=false 2024-11-20T17:47:12,025 INFO [M:0;613581fcdc40:37693 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:12,025 DEBUG [M:0;613581fcdc40:37693 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732124831928Disabling compacts and flushes for region at 1732124831928Disabling writes for close at 1732124831929 (+1 ms)Obtaining lock to block concurrent updates at 1732124831929Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732124831929Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1732124831929Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732124831930 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732124831930Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732124831951 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732124831951Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732124831953 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732124831979 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732124831979Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732124831982 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732124832002 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732124832002Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47b9b627: reopening flushed file at 1732124832006 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b5dbe0: reopening flushed file at 1732124832008 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ae21ad4: reopening flushed file at 1732124832010 (+2 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=102, compaction requested=false at 1732124832014 (+4 ms)Writing region close event to WAL at 1732124832024 (+10 ms)Closed at 1732124832024 2024-11-20T17:47:12,029 INFO [M:0;613581fcdc40:37693 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T17:47:12,029 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:47:12,029 INFO [M:0;613581fcdc40:37693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37693 2024-11-20T17:47:12,030 INFO [M:0;613581fcdc40:37693 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:47:12,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:12,141 INFO [M:0;613581fcdc40:37693 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:47:12,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37693-0x101595b58fd0000, quorum=127.0.0.1:56636, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:47:12,144 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T17:47:12,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T17:47:12,147 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:12,159 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=503 (was 504), ProcessCount=11 (was 11), AvailableMemoryMB=9824 (was 9458) - AvailableMemoryMB LEAK? -