2024-11-07 10:48:01,667 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-07 10:48:01,685 main DEBUG Took 0.016095 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 10:48:01,685 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 10:48:01,685 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 10:48:01,686 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 10:48:01,687 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,694 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 10:48:01,705 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,707 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,708 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,709 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,709 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,709 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,710 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,711 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,712 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,713 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,713 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,714 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,715 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:48:01,715 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,715 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 10:48:01,717 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:48:01,718 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 10:48:01,719 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 10:48:01,720 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 10:48:01,721 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 10:48:01,721 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 10:48:01,729 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 10:48:01,731 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 10:48:01,733 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 10:48:01,733 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 10:48:01,733 main DEBUG createAppenders(={Console}) 2024-11-07 10:48:01,734 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-07 10:48:01,734 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-07 10:48:01,734 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-07 10:48:01,735 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 10:48:01,735 main DEBUG OutputStream closed 2024-11-07 10:48:01,735 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 10:48:01,736 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 10:48:01,736 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-07 10:48:01,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 10:48:01,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 10:48:01,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 10:48:01,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 10:48:01,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 10:48:01,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 10:48:01,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 10:48:01,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 10:48:01,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 10:48:01,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 10:48:01,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 10:48:01,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 10:48:01,808 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 10:48:01,808 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 10:48:01,808 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 10:48:01,809 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 10:48:01,809 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 10:48:01,810 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 10:48:01,812 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 10:48:01,812 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-07 10:48:01,812 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 10:48:01,813 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-07T10:48:01,826 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-07 10:48:01,829 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 10:48:01,829 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T10:48:02,064 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba 2024-11-07T10:48:02,085 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1065, ProcessCount=11, AvailableMemoryMB=5448 2024-11-07T10:48:02,185 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T10:48:02,279 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:02,296 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/cluster_d12a4e59-ff65-ee4e-6a86-18eb4f7f9039, deleteOnExit=true 2024-11-07T10:48:02,394 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/cluster_d12a4e59-ff65-ee4e-6a86-18eb4f7f9039/zookeeper_0, clientPort=50922, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/cluster_d12a4e59-ff65-ee4e-6a86-18eb4f7f9039/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/cluster_d12a4e59-ff65-ee4e-6a86-18eb4f7f9039/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T10:48:02,405 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50922 2024-11-07T10:48:02,406 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:02,501 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 with version=8 2024-11-07T10:48:02,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/hbase-staging 2024-11-07T10:48:02,602 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T10:48:02,921 INFO [Time-limited test {}] client.ConnectionUtils(128): master/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:48:02,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:02,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:02,942 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:48:02,943 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:02,943 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:48:03,113 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T10:48:03,214 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T10:48:03,254 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T10:48:03,265 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T10:48:03,269 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:48:03,319 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17684 (auto-detected) 2024-11-07T10:48:03,320 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T10:48:03,344 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36849 2024-11-07T10:48:03,366 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36849 connecting to ZooKeeper ensemble=127.0.0.1:50922 2024-11-07T10:48:03,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:368490x0, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:48:03,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36849-0x10019db0ee10000 connected 2024-11-07T10:48:03,433 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:03,434 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:03,445 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:03,449 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9, hbase.cluster.distributed=false 2024-11-07T10:48:03,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:48:03,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36849 2024-11-07T10:48:03,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36849 2024-11-07T10:48:03,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36849 2024-11-07T10:48:03,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36849 2024-11-07T10:48:03,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36849 2024-11-07T10:48:03,569 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:48:03,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:03,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:03,572 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:48:03,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:03,573 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:48:03,576 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T10:48:03,580 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:48:03,581 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37261 2024-11-07T10:48:03,583 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37261 connecting to ZooKeeper ensemble=127.0.0.1:50922 2024-11-07T10:48:03,584 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:03,585 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:03,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:372610x0, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:48:03,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:372610x0, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:03,591 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37261-0x10019db0ee10001 connected 2024-11-07T10:48:03,598 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T10:48:03,613 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.cache 2024-11-07T10:48:03,619 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T10:48:03,620 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T10:48:03,621 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T10:48:03,623 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T10:48:03,625 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-07T10:48:03,628 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T10:48:03,633 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T10:48:03,633 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.persistence is complete. 2024-11-07T10:48:03,634 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T10:48:03,636 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T10:48:03,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:48:03,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37261 2024-11-07T10:48:03,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37261 2024-11-07T10:48:03,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37261 2024-11-07T10:48:03,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37261 2024-11-07T10:48:03,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37261 2024-11-07T10:48:03,671 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:48:03,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:03,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:03,672 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:48:03,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:03,673 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:48:03,673 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T10:48:03,673 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:48:03,675 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41817 2024-11-07T10:48:03,677 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41817 connecting to ZooKeeper ensemble=127.0.0.1:50922 2024-11-07T10:48:03,677 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:03,679 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:03,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:418170x0, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:48:03,684 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:418170x0, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:03,685 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41817-0x10019db0ee10002 connected 2024-11-07T10:48:03,685 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T10:48:03,689 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.cache 2024-11-07T10:48:03,689 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T10:48:03,690 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T10:48:03,690 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T10:48:03,692 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T10:48:03,692 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-07T10:48:03,692 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T10:48:03,692 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T10:48:03,693 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.persistence is complete. 2024-11-07T10:48:03,696 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T10:48:03,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T10:48:03,700 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:48:03,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41817 2024-11-07T10:48:03,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41817 2024-11-07T10:48:03,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41817 2024-11-07T10:48:03,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41817 2024-11-07T10:48:03,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41817 2024-11-07T10:48:03,736 DEBUG [M:0;10641b2c0a69:36849 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;10641b2c0a69:36849 2024-11-07T10:48:03,738 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/10641b2c0a69,36849,1730976482681 2024-11-07T10:48:03,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:03,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:03,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:03,750 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/10641b2c0a69,36849,1730976482681 2024-11-07T10:48:03,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T10:48:03,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:03,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:03,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T10:48:03,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:03,785 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T10:48:03,787 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/10641b2c0a69,36849,1730976482681 from backup master directory 2024-11-07T10:48:03,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:03,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/10641b2c0a69,36849,1730976482681 2024-11-07T10:48:03,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:03,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:03,792 WARN [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:48:03,792 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=10641b2c0a69,36849,1730976482681 2024-11-07T10:48:03,794 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T10:48:03,796 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T10:48:03,804 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/hbase.id] with ID: 862d3cb4-2858-4be8-a01b-2851a84bdef8 2024-11-07T10:48:03,804 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/.tmp/hbase.id 2024-11-07T10:48:03,810 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/hbase.id] 2024-11-07T10:48:03,848 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:03,854 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T10:48:03,875 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-07T10:48:03,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:03,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:03,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:03,908 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T10:48:03,911 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T10:48:03,966 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:48:03,972 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:48:04,035 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store 2024-11-07T10:48:04,063 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T10:48:04,066 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:04,067 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T10:48:04,067 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:04,068 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:04,069 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T10:48:04,069 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:04,069 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:04,070 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730976484067Disabling compacts and flushes for region at 1730976484067Disabling writes for close at 1730976484069 (+2 ms)Writing region close event to WAL at 1730976484069Closed at 1730976484069 2024-11-07T10:48:04,070 WARN [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/.initializing 2024-11-07T10:48:04,071 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/WALs/10641b2c0a69,36849,1730976482681 2024-11-07T10:48:04,084 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:48:04,099 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=10641b2c0a69%2C36849%2C1730976482681, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/WALs/10641b2c0a69,36849,1730976482681, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/oldWALs, maxLogs=10 2024-11-07T10:48:04,142 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/WALs/10641b2c0a69,36849,1730976482681/10641b2c0a69%2C36849%2C1730976482681.1730976484103 2024-11-07T10:48:04,144 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T10:48:04,145 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:48:04,145 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:04,149 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,150 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,199 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,228 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T10:48:04,232 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:04,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:04,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T10:48:04,239 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:04,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:04,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,245 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T10:48:04,245 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:04,245 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:04,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T10:48:04,251 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:04,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:04,252 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,257 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,258 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,262 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,263 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,266 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:48:04,269 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:04,278 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:48:04,279 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63495936, jitterRate=-0.053836822509765625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:48:04,285 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730976484164Initializing all the Stores at 1730976484165 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976484165Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976484166 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976484167 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976484167Cleaning up temporary data from old regions at 1730976484263 (+96 ms)Region opened successfully at 1730976484285 (+22 ms) 2024-11-07T10:48:04,287 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T10:48:04,331 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226a2b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:48:04,374 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T10:48:04,391 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T10:48:04,391 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T10:48:04,395 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:48:04,396 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T10:48:04,404 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-07T10:48:04,404 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T10:48:04,432 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T10:48:04,440 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T10:48:04,443 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T10:48:04,445 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T10:48:04,447 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T10:48:04,449 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T10:48:04,451 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T10:48:04,454 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T10:48:04,455 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T10:48:04,457 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T10:48:04,459 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T10:48:04,477 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T10:48:04,478 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T10:48:04,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:04,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:04,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:04,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:04,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:04,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:04,494 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=10641b2c0a69,36849,1730976482681, sessionid=0x10019db0ee10000, setting cluster-up flag (Was=false) 2024-11-07T10:48:04,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:04,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:04,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:04,514 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T10:48:04,516 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=10641b2c0a69,36849,1730976482681 2024-11-07T10:48:04,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:04,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:04,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:04,527 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T10:48:04,528 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=10641b2c0a69,36849,1730976482681 2024-11-07T10:48:04,533 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T10:48:04,604 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:48:04,614 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T10:48:04,617 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(746): ClusterId : 862d3cb4-2858-4be8-a01b-2851a84bdef8 2024-11-07T10:48:04,617 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(746): ClusterId : 862d3cb4-2858-4be8-a01b-2851a84bdef8 2024-11-07T10:48:04,620 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T10:48:04,620 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T10:48:04,622 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T10:48:04,625 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T10:48:04,625 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T10:48:04,626 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T10:48:04,626 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T10:48:04,629 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T10:48:04,629 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T10:48:04,630 DEBUG [RS:1;10641b2c0a69:41817 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f7db698, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:48:04,630 DEBUG [RS:0;10641b2c0a69:37261 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7490135a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:48:04,629 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 10641b2c0a69,36849,1730976482681 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T10:48:04,638 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:48:04,638 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:48:04,639 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:48:04,639 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:48:04,639 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/10641b2c0a69:0, corePoolSize=10, maxPoolSize=10 2024-11-07T10:48:04,639 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,640 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:48:04,640 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,644 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T10:48:04,645 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T10:48:04,645 DEBUG [RS:1;10641b2c0a69:41817 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;10641b2c0a69:41817 2024-11-07T10:48:04,648 DEBUG [RS:0;10641b2c0a69:37261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;10641b2c0a69:37261 2024-11-07T10:48:04,648 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T10:48:04,649 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T10:48:04,649 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976514648 2024-11-07T10:48:04,649 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T10:48:04,649 DEBUG [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T10:48:04,649 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T10:48:04,649 DEBUG [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T10:48:04,651 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T10:48:04,651 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:04,652 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,36849,1730976482681 with port=41817, startcode=1730976483670 2024-11-07T10:48:04,652 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,36849,1730976482681 with port=37261, startcode=1730976483528 2024-11-07T10:48:04,651 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T10:48:04,652 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T10:48:04,656 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T10:48:04,657 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T10:48:04,657 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T10:48:04,657 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T10:48:04,660 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,663 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T10:48:04,664 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T10:48:04,665 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T10:48:04,667 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T10:48:04,667 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T10:48:04,670 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.large.0-1730976484668,5,FailOnTimeoutGroup] 2024-11-07T10:48:04,670 DEBUG [RS:0;10641b2c0a69:37261 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T10:48:04,670 DEBUG [RS:1;10641b2c0a69:41817 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T10:48:04,670 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.small.0-1730976484670,5,FailOnTimeoutGroup] 2024-11-07T10:48:04,670 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,671 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T10:48:04,672 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,672 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,687 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T10:48:04,688 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 2024-11-07T10:48:04,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:04,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-07T10:48:04,708 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T10:48:04,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:04,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:04,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-07T10:48:04,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T10:48:04,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:04,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:04,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-07T10:48:04,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T10:48:04,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:04,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:04,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-07T10:48:04,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T10:48:04,729 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:04,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:04,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T10:48:04,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740 2024-11-07T10:48:04,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740 2024-11-07T10:48:04,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T10:48:04,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T10:48:04,731 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:48:04,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T10:48:04,739 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:48:04,739 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71776496, jitterRate=0.06955313682556152}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:48:04,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730976484703Initializing all the Stores at 1730976484703Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976484703Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976484703Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976484703Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976484704 (+1 ms)Cleaning up temporary data from old regions at 1730976484731 (+27 ms)Region opened successfully at 1730976484743 (+12 ms) 2024-11-07T10:48:04,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T10:48:04,743 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T10:48:04,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T10:48:04,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T10:48:04,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T10:48:04,745 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T10:48:04,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730976484743Disabling compacts and flushes for region at 1730976484743Disabling writes for close at 1730976484744 (+1 ms)Writing region close event to WAL at 1730976484744Closed at 1730976484745 (+1 ms) 2024-11-07T10:48:04,749 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T10:48:04,749 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T10:48:04,749 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39857, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T10:48:04,749 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58753, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T10:48:04,757 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36849 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 10641b2c0a69,37261,1730976483528 2024-11-07T10:48:04,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T10:48:04,760 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36849 {}] master.ServerManager(517): Registering regionserver=10641b2c0a69,37261,1730976483528 2024-11-07T10:48:04,768 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:48:04,770 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T10:48:04,772 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36849 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 10641b2c0a69,41817,1730976483670 2024-11-07T10:48:04,772 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36849 {}] master.ServerManager(517): Registering regionserver=10641b2c0a69,41817,1730976483670 2024-11-07T10:48:04,777 DEBUG [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 2024-11-07T10:48:04,777 DEBUG [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 2024-11-07T10:48:04,777 DEBUG [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T10:48:04,777 DEBUG [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T10:48:04,777 DEBUG [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T10:48:04,777 DEBUG [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T10:48:04,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:48:04,785 DEBUG [RS:0;10641b2c0a69:37261 {}] zookeeper.ZKUtil(111): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/10641b2c0a69,37261,1730976483528 2024-11-07T10:48:04,785 DEBUG [RS:1;10641b2c0a69:41817 {}] zookeeper.ZKUtil(111): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/10641b2c0a69,41817,1730976483670 2024-11-07T10:48:04,785 WARN [RS:0;10641b2c0a69:37261 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:48:04,785 WARN [RS:1;10641b2c0a69:41817 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:48:04,785 INFO [RS:0;10641b2c0a69:37261 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:48:04,785 INFO [RS:1;10641b2c0a69:41817 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:48:04,785 DEBUG [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,41817,1730976483670 2024-11-07T10:48:04,785 DEBUG [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,37261,1730976483528 2024-11-07T10:48:04,787 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [10641b2c0a69,37261,1730976483528] 2024-11-07T10:48:04,787 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [10641b2c0a69,41817,1730976483670] 2024-11-07T10:48:04,813 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T10:48:04,813 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T10:48:04,826 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T10:48:04,826 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T10:48:04,831 INFO [RS:0;10641b2c0a69:37261 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T10:48:04,831 INFO [RS:1;10641b2c0a69:41817 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T10:48:04,831 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,831 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,832 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T10:48:04,833 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T10:48:04,838 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T10:48:04,838 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T10:48:04,840 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,840 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,840 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,840 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,840 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,840 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,840 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,840 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,841 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,841 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,841 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,841 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,841 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:48:04,841 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:48:04,841 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,841 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,842 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,842 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,842 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,842 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,842 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,842 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,842 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,842 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,843 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,843 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:04,843 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:04,843 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:04,843 DEBUG [RS:0;10641b2c0a69:37261 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:04,843 DEBUG [RS:1;10641b2c0a69:41817 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:04,844 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,844 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,844 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,845 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,845 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,845 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,845 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,845 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,845 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,845 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,37261,1730976483528-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:48:04,845 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,845 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,41817,1730976483670-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:48:04,876 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T10:48:04,876 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T10:48:04,878 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,37261,1730976483528-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,878 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,41817,1730976483670-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,879 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,879 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,879 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.Replication(171): 10641b2c0a69,41817,1730976483670 started 2024-11-07T10:48:04,879 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.Replication(171): 10641b2c0a69,37261,1730976483528 started 2024-11-07T10:48:04,907 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,907 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:04,907 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(1482): Serving as 10641b2c0a69,37261,1730976483528, RpcServer on 10641b2c0a69/172.17.0.2:37261, sessionid=0x10019db0ee10001 2024-11-07T10:48:04,907 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(1482): Serving as 10641b2c0a69,41817,1730976483670, RpcServer on 10641b2c0a69/172.17.0.2:41817, sessionid=0x10019db0ee10002 2024-11-07T10:48:04,908 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T10:48:04,908 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T10:48:04,909 DEBUG [RS:0;10641b2c0a69:37261 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 10641b2c0a69,37261,1730976483528 2024-11-07T10:48:04,909 DEBUG [RS:1;10641b2c0a69:41817 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 10641b2c0a69,41817,1730976483670 2024-11-07T10:48:04,909 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,37261,1730976483528' 2024-11-07T10:48:04,909 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,41817,1730976483670' 2024-11-07T10:48:04,909 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T10:48:04,909 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T10:48:04,910 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T10:48:04,910 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T10:48:04,911 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T10:48:04,911 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T10:48:04,911 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T10:48:04,911 DEBUG [RS:1;10641b2c0a69:41817 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 10641b2c0a69,41817,1730976483670 2024-11-07T10:48:04,912 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T10:48:04,912 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,41817,1730976483670' 2024-11-07T10:48:04,912 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T10:48:04,912 DEBUG [RS:0;10641b2c0a69:37261 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 10641b2c0a69,37261,1730976483528 2024-11-07T10:48:04,912 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,37261,1730976483528' 2024-11-07T10:48:04,912 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T10:48:04,912 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T10:48:04,913 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T10:48:04,913 DEBUG [RS:1;10641b2c0a69:41817 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T10:48:04,913 INFO [RS:1;10641b2c0a69:41817 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T10:48:04,913 INFO [RS:1;10641b2c0a69:41817 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T10:48:04,913 DEBUG [RS:0;10641b2c0a69:37261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T10:48:04,914 INFO [RS:0;10641b2c0a69:37261 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T10:48:04,914 INFO [RS:0;10641b2c0a69:37261 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T10:48:04,921 WARN [10641b2c0a69:36849 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-07T10:48:05,019 INFO [RS:0;10641b2c0a69:37261 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:48:05,019 INFO [RS:1;10641b2c0a69:41817 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:48:05,020 INFO [RS:1;10641b2c0a69:41817 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=10641b2c0a69%2C41817%2C1730976483670, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,41817,1730976483670, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs, maxLogs=55 2024-11-07T10:48:05,020 INFO [RS:0;10641b2c0a69:37261 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=10641b2c0a69%2C37261%2C1730976483528, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,37261,1730976483528, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs, maxLogs=55 2024-11-07T10:48:05,044 INFO [RS:1;10641b2c0a69:41817 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,41817,1730976483670/10641b2c0a69%2C41817%2C1730976483670.1730976485024 2024-11-07T10:48:05,044 INFO [RS:0;10641b2c0a69:37261 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,37261,1730976483528/10641b2c0a69%2C37261%2C1730976483528.1730976485024 2024-11-07T10:48:05,045 DEBUG [RS:1;10641b2c0a69:41817 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T10:48:05,045 DEBUG [RS:0;10641b2c0a69:37261 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T10:48:05,174 DEBUG [10641b2c0a69:36849 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-07T10:48:05,185 DEBUG [10641b2c0a69:36849 {}] balancer.BalancerClusterState(204): Hosts are {10641b2c0a69=0} racks are {/default-rack=0} 2024-11-07T10:48:05,188 DEBUG [10641b2c0a69:36849 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T10:48:05,188 DEBUG [10641b2c0a69:36849 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T10:48:05,188 DEBUG [10641b2c0a69:36849 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T10:48:05,188 DEBUG [10641b2c0a69:36849 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T10:48:05,189 INFO [10641b2c0a69:36849 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T10:48:05,189 INFO [10641b2c0a69:36849 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T10:48:05,189 DEBUG [10641b2c0a69:36849 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T10:48:05,196 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=10641b2c0a69,41817,1730976483670 2024-11-07T10:48:05,201 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,41817,1730976483670, state=OPENING 2024-11-07T10:48:05,206 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T10:48:05,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:05,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:05,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:05,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:05,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:05,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:05,211 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:48:05,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,41817,1730976483670}] 2024-11-07T10:48:05,386 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T10:48:05,390 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55853, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T10:48:05,399 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T10:48:05,399 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:48:05,400 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T10:48:05,401 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=10641b2c0a69%2C41817%2C1730976483670.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,41817,1730976483670, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs, maxLogs=55 2024-11-07T10:48:05,421 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,41817,1730976483670/10641b2c0a69%2C41817%2C1730976483670.meta.1730976485403.meta 2024-11-07T10:48:05,422 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T10:48:05,422 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:48:05,424 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T10:48:05,427 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T10:48:05,432 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T10:48:05,436 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T10:48:05,437 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:05,437 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T10:48:05,437 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T10:48:05,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-07T10:48:05,440 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T10:48:05,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:05,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:05,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-07T10:48:05,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T10:48:05,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:05,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:05,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-07T10:48:05,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T10:48:05,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:05,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:05,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-07T10:48:05,445 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T10:48:05,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:05,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:05,445 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T10:48:05,446 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740 2024-11-07T10:48:05,447 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740 2024-11-07T10:48:05,448 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T10:48:05,448 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T10:48:05,448 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:48:05,449 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T10:48:05,450 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64330222, jitterRate=-0.04140499234199524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:48:05,450 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T10:48:05,452 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730976485438Writing region info on filesystem at 1730976485438Initializing all the Stores at 1730976485439 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976485439Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976485439Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976485439Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976485439Cleaning up temporary data from old regions at 1730976485448 (+9 ms)Running coprocessor post-open hooks at 1730976485450 (+2 ms)Region opened successfully at 1730976485451 (+1 ms) 2024-11-07T10:48:05,457 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730976485377 2024-11-07T10:48:05,468 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T10:48:05,468 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T10:48:05,469 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41817,1730976483670 2024-11-07T10:48:05,470 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,41817,1730976483670, state=OPEN 2024-11-07T10:48:05,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:48:05,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:48:05,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:48:05,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:05,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:05,475 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:05,475 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=10641b2c0a69,41817,1730976483670 2024-11-07T10:48:05,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T10:48:05,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,41817,1730976483670 in 264 msec 2024-11-07T10:48:05,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T10:48:05,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 723 msec 2024-11-07T10:48:05,486 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T10:48:05,486 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T10:48:05,510 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:48:05,512 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41817,1730976483670, seqNum=-1] 2024-11-07T10:48:05,537 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:48:05,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53323, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:48:05,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0010 sec 2024-11-07T10:48:05,562 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730976485562, completionTime=-1 2024-11-07T10:48:05,566 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-07T10:48:05,566 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:48:05,600 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T10:48:05,601 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730976545601 2024-11-07T10:48:05,601 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730976605601 2024-11-07T10:48:05,601 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-07T10:48:05,604 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:48:05,611 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,36849,1730976482681-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:05,611 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,36849,1730976482681-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:05,611 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,36849,1730976482681-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:05,613 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-10641b2c0a69:36849, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:05,614 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:05,615 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:05,621 DEBUG [master/10641b2c0a69:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T10:48:05,644 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.852sec 2024-11-07T10:48:05,646 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T10:48:05,647 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T10:48:05,648 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T10:48:05,649 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T10:48:05,649 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T10:48:05,650 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,36849,1730976482681-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:48:05,650 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,36849,1730976482681-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T10:48:05,651 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T10:48:05,652 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T10:48:05,653 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,36849,1730976482681-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:05,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T10:48:05,728 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T10:48:05,728 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T10:48:05,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 10641b2c0a69,36849,-1 for getting cluster id 2024-11-07T10:48:05,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T10:48:05,743 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '862d3cb4-2858-4be8-a01b-2851a84bdef8' 2024-11-07T10:48:05,745 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T10:48:05,745 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "862d3cb4-2858-4be8-a01b-2851a84bdef8" 2024-11-07T10:48:05,746 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f4f1c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T10:48:05,746 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [10641b2c0a69,36849,-1] 2024-11-07T10:48:05,748 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T10:48:05,750 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:05,751 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T10:48:05,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T10:48:05,754 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:48:05,760 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41817,1730976483670, seqNum=-1] 2024-11-07T10:48:05,761 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:48:05,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:48:05,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=10641b2c0a69,36849,1730976482681 2024-11-07T10:48:05,800 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T10:48:05,804 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 10641b2c0a69,36849,1730976482681 2024-11-07T10:48:05,807 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26ed5dcb 2024-11-07T10:48:05,808 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T10:48:05,810 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T10:48:05,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1730976485796', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T10:48:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1730976485796 2024-11-07T10:48:05,823 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730976485796 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T10:48:05,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1730976485796" procId is: 4 2024-11-07T10:48:05,825 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:05,827 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730976485796 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T10:48:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:48:05,841 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1730976485796-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 786e24eefbb7f16d1a1d004a13582487, NAME => 'testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1730976485796', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 2024-11-07T10:48:05,849 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730976485796-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:05,849 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730976485796-pool-0 {}] regionserver.HRegion(1722): Closing 786e24eefbb7f16d1a1d004a13582487, disabling compactions & flushes 2024-11-07T10:48:05,850 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1730976485796-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:05,850 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730976485796-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:05,850 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730976485796-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. after waiting 0 ms 2024-11-07T10:48:05,850 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730976485796-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:05,850 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1730976485796-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:05,850 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730976485796-pool-0 {}] regionserver.HRegion(1676): Region close journal for 786e24eefbb7f16d1a1d004a13582487: Waiting for close lock at 1730976485849Disabling compacts and flushes for region at 1730976485849Disabling writes for close at 1730976485850 (+1 ms)Writing region close event to WAL at 1730976485850Closed at 1730976485850 2024-11-07T10:48:05,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730976485796 execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T10:48:05,856 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1730976485851"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976485851"}]},"ts":"1730976485851"} 2024-11-07T10:48:05,860 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T10:48:05,861 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730976485796 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T10:48:05,864 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1730976485796","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730976485861"}]},"ts":"1730976485861"} 2024-11-07T10:48:05,867 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1730976485796, state=ENABLING in hbase:meta 2024-11-07T10:48:05,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {10641b2c0a69=0} racks are {/default-rack=0} 2024-11-07T10:48:05,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T10:48:05,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T10:48:05,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T10:48:05,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T10:48:05,869 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T10:48:05,869 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T10:48:05,869 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T10:48:05,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, ASSIGN}] 2024-11-07T10:48:05,871 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, ASSIGN 2024-11-07T10:48:05,872 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, ASSIGN; state=OFFLINE, location=10641b2c0a69,37261,1730976483528; forceNewPlan=false, retain=false 2024-11-07T10:48:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:48:06,025 INFO [10641b2c0a69:36849 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-07T10:48:06,026 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=786e24eefbb7f16d1a1d004a13582487, regionState=OPENING, regionLocation=10641b2c0a69,37261,1730976483528 2024-11-07T10:48:06,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, ASSIGN because future has completed 2024-11-07T10:48:06,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 786e24eefbb7f16d1a1d004a13582487, server=10641b2c0a69,37261,1730976483528}] 2024-11-07T10:48:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:48:06,183 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T10:48:06,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43535, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T10:48:06,188 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:06,189 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 786e24eefbb7f16d1a1d004a13582487, NAME => 'testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487.', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:48:06,189 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1730976485796 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,189 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:06,190 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,190 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,191 INFO [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,195 INFO [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 786e24eefbb7f16d1a1d004a13582487 columnFamilyName family 2024-11-07T10:48:06,195 DEBUG [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:06,196 INFO [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] regionserver.HStore(327): Store=786e24eefbb7f16d1a1d004a13582487/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:06,196 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,197 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,197 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,197 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,197 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,198 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,205 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:48:06,205 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 786e24eefbb7f16d1a1d004a13582487; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72008565, jitterRate=0.07301123440265656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T10:48:06,205 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:06,206 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 786e24eefbb7f16d1a1d004a13582487: Running coprocessor pre-open hook at 1730976486190Writing region info on filesystem at 1730976486190Initializing all the Stores at 1730976486190Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976486190Cleaning up temporary data from old regions at 1730976486197 (+7 ms)Running coprocessor post-open hooks at 1730976486205 (+8 ms)Region opened successfully at 1730976486206 (+1 ms) 2024-11-07T10:48:06,207 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487., pid=6, masterSystemTime=1730976486183 2024-11-07T10:48:06,210 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:06,210 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:06,211 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=786e24eefbb7f16d1a1d004a13582487, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,37261,1730976483528 2024-11-07T10:48:06,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 786e24eefbb7f16d1a1d004a13582487, server=10641b2c0a69,37261,1730976483528 because future has completed 2024-11-07T10:48:06,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T10:48:06,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 786e24eefbb7f16d1a1d004a13582487, server=10641b2c0a69,37261,1730976483528 in 185 msec 2024-11-07T10:48:06,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T10:48:06,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, ASSIGN in 347 msec 2024-11-07T10:48:06,220 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730976485796 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T10:48:06,221 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1730976485796","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730976486220"}]},"ts":"1730976486220"} 2024-11-07T10:48:06,223 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1730976485796, state=ENABLED in hbase:meta 2024-11-07T10:48:06,224 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730976485796 execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T10:48:06,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1730976485796 in 406 msec 2024-11-07T10:48:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:48:06,459 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1730976485796 get assigned. Timeout = 60000ms 2024-11-07T10:48:06,459 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1730976485796 completed 2024-11-07T10:48:06,461 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T10:48:06,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1730976485796 assigned to meta. Checking AM states. 2024-11-07T10:48:06,467 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T10:48:06,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1730976485796 assigned. 2024-11-07T10:48:06,476 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1730976485796', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487., hostname=10641b2c0a69,37261,1730976483528, seqNum=2] 2024-11-07T10:48:06,477 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:48:06,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:48:06,489 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 786e24eefbb7f16d1a1d004a13582487 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-07T10:48:06,516 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fe4c85ac2ff44dfcb66fba2b7d12792a 2024-11-07T10:48:06,553 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/.tmp/family/fe4c85ac2ff44dfcb66fba2b7d12792a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-07T10:48:06,558 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/.tmp/family/fe4c85ac2ff44dfcb66fba2b7d12792a 2024-11-07T10:48:06,592 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/.tmp/family/fe4c85ac2ff44dfcb66fba2b7d12792a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a 2024-11-07T10:48:06,594 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a, delay=910 ms 2024-11-07T10:48:06,596 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-07T10:48:06,601 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 786e24eefbb7f16d1a1d004a13582487 in 112ms, sequenceid=6, compaction requested=false 2024-11-07T10:48:06,602 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1730976485796' 2024-11-07T10:48:06,605 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 786e24eefbb7f16d1a1d004a13582487: 2024-11-07T10:48:07,506 DEBUG [hfile-prefetch-1730976486594 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fe4c85ac2ff44dfcb66fba2b7d12792a_0. 2024-11-07T10:48:07,511 DEBUG [hfile-prefetch-1730976486594 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fe4c85ac2ff44dfcb66fba2b7d12792a_117. 2024-11-07T10:48:07,512 DEBUG [hfile-prefetch-1730976486594 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a, with total blocks 2, and data blocks 1 2024-11-07T10:48:07,512 DEBUG [hfile-prefetch-1730976486594 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-07T10:48:07,512 DEBUG [hfile-prefetch-1730976486594 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-07T10:48:07,513 DEBUG [hfile-prefetch-1730976486594 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a. Will sleep 100ms and try the verification again. 2024-11-07T10:48:07,525 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-07T10:48:07,525 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-07T10:48:07,525 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T10:48:07,525 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T10:48:07,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-07T10:48:07,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1730976487525 2024-11-07T10:48:07,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-07T10:48:07,613 DEBUG [hfile-prefetch-1730976486594 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a, with total blocks 2, and data blocks 1 2024-11-07T10:48:07,613 DEBUG [hfile-prefetch-1730976486594 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-07T10:48:07,613 DEBUG [hfile-prefetch-1730976486594 {}] bucket.BucketCache(2294): found block for file fe4c85ac2ff44dfcb66fba2b7d12792a in the backing map. Acquiring read lock for offset 0 2024-11-07T10:48:07,613 DEBUG [hfile-prefetch-1730976486594 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a has now been fully cached. 2024-11-07T10:48:07,614 DEBUG [hfile-prefetch-1730976486594 {}] hfile.PrefetchExecutor(121): Prefetch completed for fe4c85ac2ff44dfcb66fba2b7d12792a 2024-11-07T10:48:07,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=786e24eefbb7f16d1a1d004a13582487, source=10641b2c0a69,37261,1730976483528, destination=10641b2c0a69,41817,1730976483670, warming up region on 10641b2c0a69,41817,1730976483670 2024-11-07T10:48:07,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=786e24eefbb7f16d1a1d004a13582487, source=10641b2c0a69,37261,1730976483528, destination=10641b2c0a69,41817,1730976483670, running balancer 2024-11-07T10:48:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, REOPEN/MOVE 2024-11-07T10:48:07,621 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, REOPEN/MOVE 2024-11-07T10:48:07,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.HRegion(7855): Warmup {ENCODED => 786e24eefbb7f16d1a1d004a13582487, NAME => 'testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487.', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:48:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:07,622 INFO [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:07,623 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=786e24eefbb7f16d1a1d004a13582487, regionState=CLOSING, regionLocation=10641b2c0a69,37261,1730976483528 2024-11-07T10:48:07,623 INFO [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 786e24eefbb7f16d1a1d004a13582487 columnFamilyName family 2024-11-07T10:48:07,624 DEBUG [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:07,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, REOPEN/MOVE because future has completed 2024-11-07T10:48:07,626 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-07T10:48:07,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 786e24eefbb7f16d1a1d004a13582487, server=10641b2c0a69,37261,1730976483528}] 2024-11-07T10:48:07,628 DEBUG [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a 2024-11-07T10:48:07,628 INFO [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] regionserver.HStore(327): Store=786e24eefbb7f16d1a1d004a13582487/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.HRegion(1722): Closing 786e24eefbb7f16d1a1d004a13582487, disabling compactions & flushes 2024-11-07T10:48:07,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. after waiting 0 ms 2024-11-07T10:48:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:07,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41817 {}] regionserver.HRegion(1676): Region close journal for 786e24eefbb7f16d1a1d004a13582487: Waiting for close lock at 1730976487628Disabling compacts and flushes for region at 1730976487628Disabling writes for close at 1730976487628Writing region close event to WAL at 1730976487630 (+2 ms)Closed at 1730976487630 2024-11-07T10:48:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36849 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-07T10:48:07,784 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:07,784 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-07T10:48:07,785 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 786e24eefbb7f16d1a1d004a13582487, disabling compactions & flushes 2024-11-07T10:48:07,785 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:07,785 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:07,785 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. after waiting 0 ms 2024-11-07T10:48:07,785 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:07,787 DEBUG [StoreFileCloser-786e24eefbb7f16d1a1d004a13582487-family-1 {}] bucket.BucketCache(769): Removing file fe4c85ac2ff44dfcb66fba2b7d12792a for region 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:07,787 DEBUG [StoreFileCloser-786e24eefbb7f16d1a1d004a13582487-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file fe4c85ac2ff44dfcb66fba2b7d12792a, starting offset: 0, end offset: 9223372036854775807 2024-11-07T10:48:07,789 DEBUG [StoreFileCloser-786e24eefbb7f16d1a1d004a13582487-family-1 {}] bucket.BucketCache(857): removed key fe4c85ac2ff44dfcb66fba2b7d12792a_0 from back map with offset lock 2094080 in the evict process 2024-11-07T10:48:07,793 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-07T10:48:07,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-07T10:48:07,794 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T10:48:07,794 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T10:48:07,794 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-07T10:48:07,795 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:07,796 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 786e24eefbb7f16d1a1d004a13582487: Waiting for close lock at 1730976487785Running coprocessor pre-close hooks at 1730976487785Disabling compacts and flushes for region at 1730976487785Disabling writes for close at 1730976487785Writing region close event to WAL at 1730976487789 (+4 ms)Running coprocessor post-close hooks at 1730976487794 (+5 ms)Closed at 1730976487795 (+1 ms) 2024-11-07T10:48:07,796 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 786e24eefbb7f16d1a1d004a13582487 move to 10641b2c0a69,41817,1730976483670 record at close sequenceid=6 2024-11-07T10:48:07,797 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T10:48:07,797 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730976487794 2024-11-07T10:48:07,797 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-07T10:48:07,800 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:07,800 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=786e24eefbb7f16d1a1d004a13582487, regionState=CLOSED 2024-11-07T10:48:07,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 786e24eefbb7f16d1a1d004a13582487, server=10641b2c0a69,37261,1730976483528 because future has completed 2024-11-07T10:48:07,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T10:48:07,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 786e24eefbb7f16d1a1d004a13582487, server=10641b2c0a69,37261,1730976483528 in 178 msec 2024-11-07T10:48:07,807 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, REOPEN/MOVE; state=CLOSED, location=10641b2c0a69,41817,1730976483670; forceNewPlan=false, retain=false 2024-11-07T10:48:07,957 INFO [10641b2c0a69:36849 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-07T10:48:07,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=786e24eefbb7f16d1a1d004a13582487, regionState=OPENING, regionLocation=10641b2c0a69,41817,1730976483670 2024-11-07T10:48:07,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, REOPEN/MOVE because future has completed 2024-11-07T10:48:07,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 786e24eefbb7f16d1a1d004a13582487, server=10641b2c0a69,41817,1730976483670}] 2024-11-07T10:48:08,117 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:08,117 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 786e24eefbb7f16d1a1d004a13582487, NAME => 'testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487.', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:48:08,118 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1730976485796 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,118 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:08,118 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,118 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,119 INFO [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,120 INFO [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 786e24eefbb7f16d1a1d004a13582487 columnFamilyName family 2024-11-07T10:48:08,121 DEBUG [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:08,123 DEBUG [StoreFileOpener-786e24eefbb7f16d1a1d004a13582487-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a, delay=958 ms 2024-11-07T10:48:08,124 DEBUG [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a 2024-11-07T10:48:08,125 INFO [StoreOpener-786e24eefbb7f16d1a1d004a13582487-1 {}] regionserver.HStore(327): Store=786e24eefbb7f16d1a1d004a13582487/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:08,125 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,126 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,126 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,126 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,126 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,131 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,131 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 786e24eefbb7f16d1a1d004a13582487; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64901107, jitterRate=-0.03289814293384552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T10:48:08,131 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,132 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 786e24eefbb7f16d1a1d004a13582487: Running coprocessor pre-open hook at 1730976488118Writing region info on filesystem at 1730976488118Initializing all the Stores at 1730976488119 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976488119Cleaning up temporary data from old regions at 1730976488126 (+7 ms)Running coprocessor post-open hooks at 1730976488131 (+5 ms)Region opened successfully at 1730976488132 (+1 ms) 2024-11-07T10:48:08,133 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487., pid=9, masterSystemTime=1730976488114 2024-11-07T10:48:08,136 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:08,136 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:08,137 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=786e24eefbb7f16d1a1d004a13582487, regionState=OPEN, openSeqNum=10, regionLocation=10641b2c0a69,41817,1730976483670 2024-11-07T10:48:08,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 786e24eefbb7f16d1a1d004a13582487, server=10641b2c0a69,41817,1730976483670 because future has completed 2024-11-07T10:48:08,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-07T10:48:08,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 786e24eefbb7f16d1a1d004a13582487, server=10641b2c0a69,41817,1730976483670 in 180 msec 2024-11-07T10:48:08,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730976485796, region=786e24eefbb7f16d1a1d004a13582487, REOPEN/MOVE in 524 msec 2024-11-07T10:48:08,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T10:48:08,173 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T10:48:08,173 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:48:08,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:08,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:08,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T10:48:08,179 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T10:48:08,179 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-07T10:48:08,179 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=10641b2c0a69,36849,1730976482681 2024-11-07T10:48:08,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:08,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:08,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:08,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,182 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:48:08,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,183 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T10:48:08,183 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:08,183 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:08,183 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:08,183 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:48:08,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:08,184 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '10641b2c0a69,37261,1730976483528' ***** 2024-11-07T10:48:08,184 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T10:48:08,184 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '10641b2c0a69,41817,1730976483670' ***** 2024-11-07T10:48:08,184 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T10:48:08,184 INFO [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T10:48:08,184 INFO [RS:1;10641b2c0a69:41817 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T10:48:08,185 INFO [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.persistence 2024-11-07T10:48:08,185 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T10:48:08,185 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T10:48:08,185 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T10:48:08,185 INFO [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T10:48:08,185 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T10:48:08,185 INFO [RS:1;10641b2c0a69:41817 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/bucket.persistence 2024-11-07T10:48:08,185 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T10:48:08,185 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T10:48:08,185 INFO [RS:1;10641b2c0a69:41817 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T10:48:08,185 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T10:48:08,185 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T10:48:08,195 DEBUG [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1488): Thread RS:0;10641b2c0a69:37261 started persisting bucket cache to file 2024-11-07T10:48:08,195 DEBUG [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T10:48:08,195 DEBUG [RS:1;10641b2c0a69:41817 {}] bucket.BucketCache(1488): Thread RS:1;10641b2c0a69:41817 started persisting bucket cache to file 2024-11-07T10:48:08,195 DEBUG [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T10:48:08,196 DEBUG [RS:1;10641b2c0a69:41817 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T10:48:08,196 DEBUG [RS:1;10641b2c0a69:41817 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T10:48:08,200 DEBUG [RS:1;10641b2c0a69:41817 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T10:48:08,200 DEBUG [RS:1;10641b2c0a69:41817 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730976488195 2024-11-07T10:48:08,200 DEBUG [RS:1;10641b2c0a69:41817 {}] bucket.BucketCache(1510): Thread RS:1;10641b2c0a69:41817 finished persisting bucket cache to file, renaming 2024-11-07T10:48:08,200 DEBUG [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T10:48:08,200 DEBUG [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730976488195 2024-11-07T10:48:08,200 DEBUG [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1510): Thread RS:0;10641b2c0a69:37261 finished persisting bucket cache to file, renaming 2024-11-07T10:48:08,200 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T10:48:08,200 WARN [RS:0;10641b2c0a69:37261 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-07T10:48:08,200 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T10:48:08,201 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T10:48:08,201 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T10:48:08,201 INFO [RS:1;10641b2c0a69:41817 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T10:48:08,201 INFO [RS:0;10641b2c0a69:37261 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T10:48:08,201 INFO [RS:0;10641b2c0a69:37261 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T10:48:08,201 INFO [RS:1;10641b2c0a69:41817 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T10:48:08,201 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(959): stopping server 10641b2c0a69,37261,1730976483528 2024-11-07T10:48:08,201 INFO [RS:0;10641b2c0a69:37261 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:48:08,201 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(3091): Received CLOSE for 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,201 INFO [RS:0;10641b2c0a69:37261 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;10641b2c0a69:37261. 2024-11-07T10:48:08,201 DEBUG [RS:0;10641b2c0a69:37261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:48:08,201 DEBUG [RS:0;10641b2c0a69:37261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:08,201 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(976): stopping server 10641b2c0a69,37261,1730976483528; all regions closed. 2024-11-07T10:48:08,202 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(959): stopping server 10641b2c0a69,41817,1730976483670 2024-11-07T10:48:08,202 INFO [RS:1;10641b2c0a69:41817 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:48:08,202 INFO [RS:1;10641b2c0a69:41817 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;10641b2c0a69:41817. 2024-11-07T10:48:08,202 DEBUG [RS:1;10641b2c0a69:41817 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:48:08,202 DEBUG [RS:1;10641b2c0a69:41817 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:08,202 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T10:48:08,202 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T10:48:08,202 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T10:48:08,202 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T10:48:08,203 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-07T10:48:08,203 DEBUG [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(1325): Online Regions={786e24eefbb7f16d1a1d004a13582487=testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487., 1588230740=hbase:meta,,1.1588230740} 2024-11-07T10:48:08,203 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T10:48:08,203 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T10:48:08,203 DEBUG [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 786e24eefbb7f16d1a1d004a13582487 2024-11-07T10:48:08,203 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 786e24eefbb7f16d1a1d004a13582487, disabling compactions & flushes 2024-11-07T10:48:08,203 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T10:48:08,203 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:08,203 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T10:48:08,203 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:08,204 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T10:48:08,204 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. after waiting 0 ms 2024-11-07T10:48:08,204 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T10:48:08,204 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:08,204 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T10:48:08,204 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-07T10:48:08,208 DEBUG [StoreFileCloser-786e24eefbb7f16d1a1d004a13582487-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/family/fe4c85ac2ff44dfcb66fba2b7d12792a 2024-11-07T10:48:08,210 DEBUG [RS:0;10641b2c0a69:37261 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs 2024-11-07T10:48:08,210 INFO [RS:0;10641b2c0a69:37261 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C37261%2C1730976483528:(num 1730976485024) 2024-11-07T10:48:08,210 DEBUG [RS:0;10641b2c0a69:37261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:08,211 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:08,211 INFO [RS:0;10641b2c0a69:37261 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:48:08,211 INFO [RS:0;10641b2c0a69:37261 {}] hbase.ChoreService(370): Chore service for: regionserver/10641b2c0a69:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T10:48:08,211 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T10:48:08,211 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T10:48:08,211 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T10:48:08,211 INFO [RS:0;10641b2c0a69:37261 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:48:08,212 INFO [regionserver/10641b2c0a69:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:48:08,212 INFO [RS:0;10641b2c0a69:37261 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37261 2024-11-07T10:48:08,215 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnRegionMove1730976485796/786e24eefbb7f16d1a1d004a13582487/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-07T10:48:08,216 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:08,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:48:08,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/10641b2c0a69,37261,1730976483528 2024-11-07T10:48:08,216 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 786e24eefbb7f16d1a1d004a13582487: Waiting for close lock at 1730976488203Running coprocessor pre-close hooks at 1730976488203Disabling compacts and flushes for region at 1730976488203Disabling writes for close at 1730976488204 (+1 ms)Writing region close event to WAL at 1730976488208 (+4 ms)Running coprocessor post-close hooks at 1730976488216 (+8 ms)Closed at 1730976488216 2024-11-07T10:48:08,216 INFO [RS:0;10641b2c0a69:37261 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:48:08,217 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487. 2024-11-07T10:48:08,217 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [10641b2c0a69,37261,1730976483528] 2024-11-07T10:48:08,218 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1bffa7bdef9d40ce86feda78c5f3f6db 2024-11-07T10:48:08,219 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/10641b2c0a69,37261,1730976483528 already deleted, retry=false 2024-11-07T10:48:08,220 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 10641b2c0a69,37261,1730976483528 expired; onlineServers=1 2024-11-07T10:48:08,242 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/info/1bffa7bdef9d40ce86feda78c5f3f6db is 205, key is testBlockEvictionOnRegionMove1730976485796,,1730976485812.786e24eefbb7f16d1a1d004a13582487./info:regioninfo/1730976488136/Put/seqid=0 2024-11-07T10:48:08,243 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/info/1bffa7bdef9d40ce86feda78c5f3f6db 2024-11-07T10:48:08,250 INFO [regionserver/10641b2c0a69:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:08,250 INFO [regionserver/10641b2c0a69:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:08,254 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f8f3559144384b3dad4730c4f25782ec 2024-11-07T10:48:08,267 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/ns/f8f3559144384b3dad4730c4f25782ec is 43, key is default/ns:d/1730976485546/Put/seqid=0 2024-11-07T10:48:08,268 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/ns/f8f3559144384b3dad4730c4f25782ec 2024-11-07T10:48:08,278 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5f331d1e5f22454c87ef4dc09d4850e7 2024-11-07T10:48:08,290 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/table/5f331d1e5f22454c87ef4dc09d4850e7 is 78, key is testBlockEvictionOnRegionMove1730976485796/table:state/1730976486220/Put/seqid=0 2024-11-07T10:48:08,291 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/table/5f331d1e5f22454c87ef4dc09d4850e7 2024-11-07T10:48:08,294 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/info/1bffa7bdef9d40ce86feda78c5f3f6db as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/info/1bffa7bdef9d40ce86feda78c5f3f6db 2024-11-07T10:48:08,295 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/info/1bffa7bdef9d40ce86feda78c5f3f6db, delay=904 ms 2024-11-07T10:48:08,296 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/info/1bffa7bdef9d40ce86feda78c5f3f6db, entries=15, sequenceid=15, filesize=7.6 K 2024-11-07T10:48:08,297 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/ns/f8f3559144384b3dad4730c4f25782ec as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/ns/f8f3559144384b3dad4730c4f25782ec 2024-11-07T10:48:08,298 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/ns/f8f3559144384b3dad4730c4f25782ec, delay=960 ms 2024-11-07T10:48:08,300 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/ns/f8f3559144384b3dad4730c4f25782ec, entries=2, sequenceid=15, filesize=5.0 K 2024-11-07T10:48:08,301 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/table/5f331d1e5f22454c87ef4dc09d4850e7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/table/5f331d1e5f22454c87ef4dc09d4850e7 2024-11-07T10:48:08,302 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/table/5f331d1e5f22454c87ef4dc09d4850e7, delay=933 ms 2024-11-07T10:48:08,303 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/table/5f331d1e5f22454c87ef4dc09d4850e7, entries=2, sequenceid=15, filesize=5.3 K 2024-11-07T10:48:08,304 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=15, compaction requested=false 2024-11-07T10:48:08,305 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T10:48:08,306 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/info/1bffa7bdef9d40ce86feda78c5f3f6db 2024-11-07T10:48:08,307 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/ns/f8f3559144384b3dad4730c4f25782ec 2024-11-07T10:48:08,308 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/table/5f331d1e5f22454c87ef4dc09d4850e7 2024-11-07T10:48:08,313 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-07T10:48:08,313 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T10:48:08,314 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T10:48:08,314 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730976488203Running coprocessor pre-close hooks at 1730976488203Disabling compacts and flushes for region at 1730976488203Disabling writes for close at 1730976488204 (+1 ms)Obtaining lock to block concurrent updates at 1730976488204Preparing flush snapshotting stores in 1588230740 at 1730976488204Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1730976488204Flushing stores of hbase:meta,,1.1588230740 at 1730976488205 (+1 ms)Flushing 1588230740/info: creating writer at 1730976488205Flushing 1588230740/info: appending metadata at 1730976488241 (+36 ms)Flushing 1588230740/info: closing flushed file at 1730976488242 (+1 ms)Flushing 1588230740/ns: creating writer at 1730976488247 (+5 ms)Flushing 1588230740/ns: appending metadata at 1730976488267 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1730976488267Flushing 1588230740/table: creating writer at 1730976488271 (+4 ms)Flushing 1588230740/table: appending metadata at 1730976488290 (+19 ms)Flushing 1588230740/table: closing flushed file at 1730976488290Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@505c04e2: reopening flushed file at 1730976488293 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a1b6caf: reopening flushed file at 1730976488297 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30094ee1: reopening flushed file at 1730976488300 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=15, compaction requested=false at 1730976488304 (+4 ms)Writing region close event to WAL at 1730976488308 (+4 ms)Running coprocessor post-close hooks at 1730976488313 (+5 ms)Closed at 1730976488314 (+1 ms) 2024-11-07T10:48:08,314 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T10:48:08,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:08,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37261-0x10019db0ee10001, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:08,319 INFO [RS:0;10641b2c0a69:37261 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:48:08,320 INFO [RS:0;10641b2c0a69:37261 {}] regionserver.HRegionServer(1031): Exiting; stopping=10641b2c0a69,37261,1730976483528; zookeeper connection closed. 2024-11-07T10:48:08,320 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11cbd564 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11cbd564 2024-11-07T10:48:08,403 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(976): stopping server 10641b2c0a69,41817,1730976483670; all regions closed. 2024-11-07T10:48:08,406 DEBUG [RS:1;10641b2c0a69:41817 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs 2024-11-07T10:48:08,406 INFO [RS:1;10641b2c0a69:41817 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C41817%2C1730976483670.meta:.meta(num 1730976485403) 2024-11-07T10:48:08,408 DEBUG [RS:1;10641b2c0a69:41817 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs 2024-11-07T10:48:08,408 INFO [RS:1;10641b2c0a69:41817 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C41817%2C1730976483670:(num 1730976485024) 2024-11-07T10:48:08,408 DEBUG [RS:1;10641b2c0a69:41817 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:08,408 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:08,408 INFO [RS:1;10641b2c0a69:41817 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:48:08,409 INFO [RS:1;10641b2c0a69:41817 {}] hbase.ChoreService(370): Chore service for: regionserver/10641b2c0a69:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T10:48:08,409 INFO [RS:1;10641b2c0a69:41817 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:48:08,409 INFO [regionserver/10641b2c0a69:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:48:08,409 INFO [RS:1;10641b2c0a69:41817 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41817 2024-11-07T10:48:08,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/10641b2c0a69,41817,1730976483670 2024-11-07T10:48:08,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:48:08,411 INFO [RS:1;10641b2c0a69:41817 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:48:08,413 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [10641b2c0a69,41817,1730976483670] 2024-11-07T10:48:08,415 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/10641b2c0a69,41817,1730976483670 already deleted, retry=false 2024-11-07T10:48:08,415 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 10641b2c0a69,41817,1730976483670 expired; onlineServers=0 2024-11-07T10:48:08,415 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '10641b2c0a69,36849,1730976482681' ***** 2024-11-07T10:48:08,415 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T10:48:08,415 INFO [M:0;10641b2c0a69:36849 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:48:08,415 INFO [M:0;10641b2c0a69:36849 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:48:08,415 DEBUG [M:0;10641b2c0a69:36849 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T10:48:08,415 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T10:48:08,415 DEBUG [M:0;10641b2c0a69:36849 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T10:48:08,415 DEBUG [master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.small.0-1730976484670 {}] cleaner.HFileCleaner(306): Exit Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.small.0-1730976484670,5,FailOnTimeoutGroup] 2024-11-07T10:48:08,416 DEBUG [master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.large.0-1730976484668 {}] cleaner.HFileCleaner(306): Exit Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.large.0-1730976484668,5,FailOnTimeoutGroup] 2024-11-07T10:48:08,416 INFO [M:0;10641b2c0a69:36849 {}] hbase.ChoreService(370): Chore service for: master/10641b2c0a69:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T10:48:08,416 INFO [M:0;10641b2c0a69:36849 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:48:08,416 DEBUG [M:0;10641b2c0a69:36849 {}] master.HMaster(1795): Stopping service threads 2024-11-07T10:48:08,416 INFO [M:0;10641b2c0a69:36849 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T10:48:08,416 INFO [M:0;10641b2c0a69:36849 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:48:08,417 INFO [M:0;10641b2c0a69:36849 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:48:08,417 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T10:48:08,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T10:48:08,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,418 DEBUG [M:0;10641b2c0a69:36849 {}] zookeeper.ZKUtil(347): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T10:48:08,418 WARN [M:0;10641b2c0a69:36849 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T10:48:08,418 INFO [M:0;10641b2c0a69:36849 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/.lastflushedseqids 2024-11-07T10:48:08,433 INFO [M:0;10641b2c0a69:36849 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:48:08,433 INFO [M:0;10641b2c0a69:36849 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T10:48:08,433 DEBUG [M:0;10641b2c0a69:36849 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T10:48:08,433 INFO [M:0;10641b2c0a69:36849 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:08,433 DEBUG [M:0;10641b2c0a69:36849 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:08,433 DEBUG [M:0;10641b2c0a69:36849 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T10:48:08,433 DEBUG [M:0;10641b2c0a69:36849 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:08,433 INFO [M:0;10641b2c0a69:36849 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-07T10:48:08,446 DEBUG [M:0;10641b2c0a69:36849 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e0169121045b4c779465a7e3e3ac1745 2024-11-07T10:48:08,459 DEBUG [M:0;10641b2c0a69:36849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e0169121045b4c779465a7e3e3ac1745 is 82, key is hbase:meta,,1/info:regioninfo/1730976485469/Put/seqid=0 2024-11-07T10:48:08,460 INFO [M:0;10641b2c0a69:36849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e0169121045b4c779465a7e3e3ac1745 2024-11-07T10:48:08,469 DEBUG [M:0;10641b2c0a69:36849 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f94823ed24a0471b99ab20c754173ce1 2024-11-07T10:48:08,482 DEBUG [M:0;10641b2c0a69:36849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f94823ed24a0471b99ab20c754173ce1 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1730976486226/Put/seqid=0 2024-11-07T10:48:08,483 INFO [M:0;10641b2c0a69:36849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f94823ed24a0471b99ab20c754173ce1 2024-11-07T10:48:08,491 DEBUG [M:0;10641b2c0a69:36849 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4897737302ca4e6287d80016c2113686 2024-11-07T10:48:08,503 DEBUG [M:0;10641b2c0a69:36849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4897737302ca4e6287d80016c2113686 is 69, key is 10641b2c0a69,37261,1730976483528/rs:state/1730976484762/Put/seqid=0 2024-11-07T10:48:08,504 INFO [M:0;10641b2c0a69:36849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4897737302ca4e6287d80016c2113686 2024-11-07T10:48:08,506 DEBUG [M:0;10641b2c0a69:36849 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e0169121045b4c779465a7e3e3ac1745 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e0169121045b4c779465a7e3e3ac1745 2024-11-07T10:48:08,508 INFO [M:0;10641b2c0a69:36849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e0169121045b4c779465a7e3e3ac1745, entries=8, sequenceid=84, filesize=5.5 K 2024-11-07T10:48:08,508 DEBUG [M:0;10641b2c0a69:36849 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f94823ed24a0471b99ab20c754173ce1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f94823ed24a0471b99ab20c754173ce1 2024-11-07T10:48:08,510 INFO [M:0;10641b2c0a69:36849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f94823ed24a0471b99ab20c754173ce1, entries=9, sequenceid=84, filesize=6.5 K 2024-11-07T10:48:08,511 DEBUG [M:0;10641b2c0a69:36849 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4897737302ca4e6287d80016c2113686 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4897737302ca4e6287d80016c2113686 2024-11-07T10:48:08,513 INFO [M:0;10641b2c0a69:36849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4897737302ca4e6287d80016c2113686, entries=2, sequenceid=84, filesize=5.1 K 2024-11-07T10:48:08,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:08,513 INFO [RS:1;10641b2c0a69:41817 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:48:08,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41817-0x10019db0ee10002, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:08,514 INFO [RS:1;10641b2c0a69:41817 {}] regionserver.HRegionServer(1031): Exiting; stopping=10641b2c0a69,41817,1730976483670; zookeeper connection closed. 2024-11-07T10:48:08,514 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28660c2d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28660c2d 2024-11-07T10:48:08,514 INFO [M:0;10641b2c0a69:36849 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false 2024-11-07T10:48:08,514 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-07T10:48:08,516 INFO [M:0;10641b2c0a69:36849 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:08,516 DEBUG [M:0;10641b2c0a69:36849 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730976488433Disabling compacts and flushes for region at 1730976488433Disabling writes for close at 1730976488433Obtaining lock to block concurrent updates at 1730976488433Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730976488433Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1730976488434 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730976488434Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730976488434Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730976488459 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730976488459Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730976488463 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730976488482 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730976488482Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730976488485 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730976488503 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730976488503Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5486f2da: reopening flushed file at 1730976488505 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bf865de: reopening flushed file at 1730976488508 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2912ac89: reopening flushed file at 1730976488511 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false at 1730976488514 (+3 ms)Writing region close event to WAL at 1730976488516 (+2 ms)Closed at 1730976488516 2024-11-07T10:48:08,518 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:48:08,518 INFO [M:0;10641b2c0a69:36849 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T10:48:08,518 INFO [M:0;10641b2c0a69:36849 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36849 2024-11-07T10:48:08,518 INFO [M:0;10641b2c0a69:36849 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:48:08,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:08,621 INFO [M:0;10641b2c0a69:36849 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:48:08,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36849-0x10019db0ee10000, quorum=127.0.0.1:50922, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:08,626 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T10:48:08,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-07T10:48:08,649 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:08,659 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/10641b2c0a69:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1730976488123 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/10641b2c0a69:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/10641b2c0a69:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1730976488298 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1730976488295 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1730976486594 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/10641b2c0a69:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=947 (was 1065), ProcessCount=11 (was 11), AvailableMemoryMB=5109 (was 5448) 2024-11-07T10:48:08,666 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=947, ProcessCount=12, AvailableMemoryMB=5108 2024-11-07T10:48:08,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/hadoop-log-dir so I do NOT create it in target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6 2024-11-07T10:48:08,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/hadoop-tmp-dir so I do NOT create it in target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6 2024-11-07T10:48:08,667 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:08,670 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/cluster_335ada61-bc20-ffca-c9b1-d6ab7bfb27b9, deleteOnExit=true 2024-11-07T10:48:08,675 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/cluster_335ada61-bc20-ffca-c9b1-d6ab7bfb27b9/zookeeper_0, clientPort=53542, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/cluster_335ada61-bc20-ffca-c9b1-d6ab7bfb27b9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/cluster_335ada61-bc20-ffca-c9b1-d6ab7bfb27b9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T10:48:08,676 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53542 2024-11-07T10:48:08,687 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 with version=8 2024-11-07T10:48:08,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/hbase-staging 2024-11-07T10:48:08,690 INFO [Time-limited test {}] client.ConnectionUtils(128): master/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:48:08,690 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:08,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:08,691 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:48:08,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:08,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:48:08,691 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T10:48:08,691 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:48:08,692 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42249 2024-11-07T10:48:08,693 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42249 connecting to ZooKeeper ensemble=127.0.0.1:53542 2024-11-07T10:48:08,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:422490x0, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:48:08,700 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42249-0x10019db27800000 connected 2024-11-07T10:48:08,722 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:08,723 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:08,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:08,724 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9, hbase.cluster.distributed=false 2024-11-07T10:48:08,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:48:08,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42249 2024-11-07T10:48:08,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42249 2024-11-07T10:48:08,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42249 2024-11-07T10:48:08,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42249 2024-11-07T10:48:08,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42249 2024-11-07T10:48:08,749 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:48:08,749 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:08,749 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:08,749 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:48:08,749 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:08,749 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:48:08,750 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T10:48:08,750 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:48:08,751 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42587 2024-11-07T10:48:08,752 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42587 connecting to ZooKeeper ensemble=127.0.0.1:53542 2024-11-07T10:48:08,753 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:08,753 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:08,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425870x0, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:48:08,757 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42587-0x10019db27800001 connected 2024-11-07T10:48:08,757 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:08,758 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T10:48:08,759 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.cache 2024-11-07T10:48:08,759 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T10:48:08,759 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T10:48:08,759 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T10:48:08,760 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T10:48:08,760 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T10:48:08,760 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-07T10:48:08,760 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T10:48:08,760 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.persistence is complete. 2024-11-07T10:48:08,760 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T10:48:08,761 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T10:48:08,762 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:48:08,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42587 2024-11-07T10:48:08,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42587 2024-11-07T10:48:08,769 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42587 2024-11-07T10:48:08,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42587 2024-11-07T10:48:08,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42587 2024-11-07T10:48:08,787 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:48:08,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:08,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:08,788 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:48:08,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:08,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:48:08,788 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T10:48:08,788 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:48:08,789 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32933 2024-11-07T10:48:08,790 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32933 connecting to ZooKeeper ensemble=127.0.0.1:53542 2024-11-07T10:48:08,790 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:08,791 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:08,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329330x0, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:48:08,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32933-0x10019db27800002 connected 2024-11-07T10:48:08,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:08,795 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T10:48:08,796 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.cache 2024-11-07T10:48:08,796 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T10:48:08,796 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T10:48:08,796 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T10:48:08,797 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T10:48:08,797 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-07T10:48:08,797 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T10:48:08,797 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.persistence is complete. 2024-11-07T10:48:08,800 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T10:48:08,801 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T10:48:08,802 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T10:48:08,803 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:48:08,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32933 2024-11-07T10:48:08,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32933 2024-11-07T10:48:08,809 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32933 2024-11-07T10:48:08,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32933 2024-11-07T10:48:08,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32933 2024-11-07T10:48:08,825 DEBUG [M:0;10641b2c0a69:42249 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;10641b2c0a69:42249 2024-11-07T10:48:08,825 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/10641b2c0a69,42249,1730976488690 2024-11-07T10:48:08,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:08,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:08,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:08,829 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/10641b2c0a69,42249,1730976488690 2024-11-07T10:48:08,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T10:48:08,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T10:48:08,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,832 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T10:48:08,832 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/10641b2c0a69,42249,1730976488690 from backup master directory 2024-11-07T10:48:08,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/10641b2c0a69,42249,1730976488690 2024-11-07T10:48:08,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:08,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:08,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:48:08,834 WARN [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:48:08,834 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=10641b2c0a69,42249,1730976488690 2024-11-07T10:48:08,836 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/hbase.id] with ID: 75843e14-39e7-4ded-9e6c-322c5bc461ae 2024-11-07T10:48:08,836 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/.tmp/hbase.id 2024-11-07T10:48:08,845 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/hbase.id] 2024-11-07T10:48:08,885 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:08,885 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T10:48:08,888 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-07T10:48:08,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:08,895 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T10:48:08,897 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T10:48:08,897 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:48:08,925 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store 2024-11-07T10:48:08,931 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:08,932 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T10:48:08,932 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:08,932 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:08,932 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T10:48:08,932 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:08,932 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:08,932 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730976488932Disabling compacts and flushes for region at 1730976488932Disabling writes for close at 1730976488932Writing region close event to WAL at 1730976488932Closed at 1730976488932 2024-11-07T10:48:08,932 WARN [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/.initializing 2024-11-07T10:48:08,933 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/WALs/10641b2c0a69,42249,1730976488690 2024-11-07T10:48:08,939 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:48:08,942 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=10641b2c0a69%2C42249%2C1730976488690, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/WALs/10641b2c0a69,42249,1730976488690, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/oldWALs, maxLogs=10 2024-11-07T10:48:08,963 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/WALs/10641b2c0a69,42249,1730976488690/10641b2c0a69%2C42249%2C1730976488690.1730976488943 2024-11-07T10:48:08,963 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T10:48:08,964 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:48:08,964 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:08,964 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,964 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T10:48:08,970 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:08,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:08,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T10:48:08,974 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:08,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:08,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T10:48:08,978 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:08,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:08,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,981 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T10:48:08,982 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:08,982 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:08,982 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,982 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,982 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,983 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,983 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,983 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:48:08,983 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:48:08,993 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:48:08,993 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62393344, jitterRate=-0.0702667236328125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:48:08,994 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730976488964Initializing all the Stores at 1730976488965 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976488965Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976488965Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976488965Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976488965Cleaning up temporary data from old regions at 1730976488983 (+18 ms)Region opened successfully at 1730976488994 (+11 ms) 2024-11-07T10:48:08,994 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T10:48:08,998 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9acbdb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:48:08,999 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T10:48:09,000 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T10:48:09,000 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T10:48:09,000 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:48:09,000 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T10:48:09,001 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-07T10:48:09,001 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T10:48:09,002 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T10:48:09,003 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T10:48:09,005 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T10:48:09,005 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T10:48:09,006 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T10:48:09,007 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T10:48:09,007 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T10:48:09,008 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T10:48:09,010 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T10:48:09,010 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T10:48:09,012 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T10:48:09,014 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T10:48:09,015 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T10:48:09,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:09,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:09,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:09,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,018 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=10641b2c0a69,42249,1730976488690, sessionid=0x10019db27800000, setting cluster-up flag (Was=false) 2024-11-07T10:48:09,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,027 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T10:48:09,029 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=10641b2c0a69,42249,1730976488690 2024-11-07T10:48:09,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,037 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T10:48:09,038 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=10641b2c0a69,42249,1730976488690 2024-11-07T10:48:09,039 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T10:48:09,041 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:48:09,041 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T10:48:09,042 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T10:48:09,042 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 10641b2c0a69,42249,1730976488690 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T10:48:09,044 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:48:09,044 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:48:09,044 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:48:09,044 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:48:09,044 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/10641b2c0a69:0, corePoolSize=10, maxPoolSize=10 2024-11-07T10:48:09,044 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,045 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:48:09,045 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,047 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T10:48:09,047 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T10:48:09,048 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,048 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T10:48:09,058 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976519058 2024-11-07T10:48:09,058 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T10:48:09,058 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T10:48:09,058 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T10:48:09,058 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T10:48:09,058 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T10:48:09,058 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T10:48:09,059 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,059 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T10:48:09,060 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T10:48:09,060 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T10:48:09,060 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T10:48:09,060 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T10:48:09,064 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.large.0-1730976489060,5,FailOnTimeoutGroup] 2024-11-07T10:48:09,067 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.small.0-1730976489064,5,FailOnTimeoutGroup] 2024-11-07T10:48:09,067 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,067 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T10:48:09,067 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,067 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,076 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T10:48:09,076 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 2024-11-07T10:48:09,083 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:09,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-07T10:48:09,087 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T10:48:09,087 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:09,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-07T10:48:09,091 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T10:48:09,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:09,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-07T10:48:09,095 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T10:48:09,095 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:09,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-07T10:48:09,099 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T10:48:09,099 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,099 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:09,099 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T10:48:09,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740 2024-11-07T10:48:09,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740 2024-11-07T10:48:09,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T10:48:09,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T10:48:09,101 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:48:09,101 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T10:48:09,110 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:48:09,111 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73351127, jitterRate=0.09301696717739105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:48:09,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730976489083Initializing all the Stores at 1730976489083Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976489084 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976489084Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976489084Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976489084Cleaning up temporary data from old regions at 1730976489100 (+16 ms)Region opened successfully at 1730976489112 (+12 ms) 2024-11-07T10:48:09,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T10:48:09,112 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T10:48:09,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T10:48:09,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T10:48:09,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T10:48:09,112 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T10:48:09,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730976489112Disabling compacts and flushes for region at 1730976489112Disabling writes for close at 1730976489112Writing region close event to WAL at 1730976489112Closed at 1730976489112 2024-11-07T10:48:09,113 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T10:48:09,113 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T10:48:09,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T10:48:09,115 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(746): ClusterId : 75843e14-39e7-4ded-9e6c-322c5bc461ae 2024-11-07T10:48:09,115 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:48:09,115 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T10:48:09,116 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(746): ClusterId : 75843e14-39e7-4ded-9e6c-322c5bc461ae 2024-11-07T10:48:09,116 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T10:48:09,116 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T10:48:09,118 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T10:48:09,118 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T10:48:09,119 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T10:48:09,119 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T10:48:09,121 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T10:48:09,122 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T10:48:09,122 DEBUG [RS:0;10641b2c0a69:42587 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@271ddcf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:48:09,122 DEBUG [RS:1;10641b2c0a69:32933 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf4f649, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:48:09,141 DEBUG [RS:0;10641b2c0a69:42587 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;10641b2c0a69:42587 2024-11-07T10:48:09,141 DEBUG [RS:1;10641b2c0a69:32933 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;10641b2c0a69:32933 2024-11-07T10:48:09,141 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T10:48:09,141 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T10:48:09,141 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T10:48:09,141 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T10:48:09,141 DEBUG [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T10:48:09,142 DEBUG [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T10:48:09,143 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,42249,1730976488690 with port=32933, startcode=1730976488787 2024-11-07T10:48:09,143 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,42249,1730976488690 with port=42587, startcode=1730976488749 2024-11-07T10:48:09,143 DEBUG [RS:0;10641b2c0a69:42587 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T10:48:09,143 DEBUG [RS:1;10641b2c0a69:32933 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T10:48:09,147 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48035, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T10:48:09,147 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42249 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 10641b2c0a69,32933,1730976488787 2024-11-07T10:48:09,148 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42249 {}] master.ServerManager(517): Registering regionserver=10641b2c0a69,32933,1730976488787 2024-11-07T10:48:09,148 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55443, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T10:48:09,149 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42249 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 10641b2c0a69,42587,1730976488749 2024-11-07T10:48:09,149 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42249 {}] master.ServerManager(517): Registering regionserver=10641b2c0a69,42587,1730976488749 2024-11-07T10:48:09,150 DEBUG [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 2024-11-07T10:48:09,150 DEBUG [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T10:48:09,150 DEBUG [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T10:48:09,151 DEBUG [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 2024-11-07T10:48:09,151 DEBUG [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T10:48:09,151 DEBUG [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T10:48:09,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:48:09,157 DEBUG [RS:1;10641b2c0a69:32933 {}] zookeeper.ZKUtil(111): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/10641b2c0a69,32933,1730976488787 2024-11-07T10:48:09,157 WARN [RS:1;10641b2c0a69:32933 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:48:09,157 INFO [RS:1;10641b2c0a69:32933 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:48:09,157 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [10641b2c0a69,42587,1730976488749] 2024-11-07T10:48:09,157 DEBUG [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,32933,1730976488787 2024-11-07T10:48:09,157 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [10641b2c0a69,32933,1730976488787] 2024-11-07T10:48:09,158 DEBUG [RS:0;10641b2c0a69:42587 {}] zookeeper.ZKUtil(111): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/10641b2c0a69,42587,1730976488749 2024-11-07T10:48:09,159 WARN [RS:0;10641b2c0a69:42587 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:48:09,159 INFO [RS:0;10641b2c0a69:42587 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:48:09,159 DEBUG [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,42587,1730976488749 2024-11-07T10:48:09,167 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T10:48:09,169 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T10:48:09,171 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T10:48:09,173 INFO [RS:1;10641b2c0a69:32933 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T10:48:09,173 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,175 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T10:48:09,175 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T10:48:09,175 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T10:48:09,175 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,175 INFO [RS:0;10641b2c0a69:42587 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T10:48:09,175 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,176 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,177 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:09,177 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T10:48:09,177 DEBUG [RS:1;10641b2c0a69:32933 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:09,177 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,177 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,177 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,177 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,177 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,177 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,178 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:48:09,178 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,178 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,178 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,178 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,178 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,178 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:09,178 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:09,179 DEBUG [RS:0;10641b2c0a69:42587 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:09,180 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,180 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,180 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,180 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,180 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,180 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,180 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,180 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,181 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,181 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,32933,1730976488787-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:48:09,181 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,181 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,42587,1730976488749-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:48:09,199 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T10:48:09,199 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,32933,1730976488787-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,199 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,199 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.Replication(171): 10641b2c0a69,32933,1730976488787 started 2024-11-07T10:48:09,206 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T10:48:09,206 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,42587,1730976488749-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,206 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,206 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.Replication(171): 10641b2c0a69,42587,1730976488749 started 2024-11-07T10:48:09,219 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,220 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(1482): Serving as 10641b2c0a69,32933,1730976488787, RpcServer on 10641b2c0a69/172.17.0.2:32933, sessionid=0x10019db27800002 2024-11-07T10:48:09,220 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T10:48:09,220 DEBUG [RS:1;10641b2c0a69:32933 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 10641b2c0a69,32933,1730976488787 2024-11-07T10:48:09,220 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,32933,1730976488787' 2024-11-07T10:48:09,220 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T10:48:09,221 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T10:48:09,221 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T10:48:09,221 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T10:48:09,221 DEBUG [RS:1;10641b2c0a69:32933 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 10641b2c0a69,32933,1730976488787 2024-11-07T10:48:09,221 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,32933,1730976488787' 2024-11-07T10:48:09,222 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T10:48:09,222 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T10:48:09,223 DEBUG [RS:1;10641b2c0a69:32933 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T10:48:09,223 INFO [RS:1;10641b2c0a69:32933 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T10:48:09,223 INFO [RS:1;10641b2c0a69:32933 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T10:48:09,229 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,229 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(1482): Serving as 10641b2c0a69,42587,1730976488749, RpcServer on 10641b2c0a69/172.17.0.2:42587, sessionid=0x10019db27800001 2024-11-07T10:48:09,229 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T10:48:09,229 DEBUG [RS:0;10641b2c0a69:42587 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 10641b2c0a69,42587,1730976488749 2024-11-07T10:48:09,229 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,42587,1730976488749' 2024-11-07T10:48:09,229 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T10:48:09,230 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T10:48:09,231 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T10:48:09,231 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T10:48:09,231 DEBUG [RS:0;10641b2c0a69:42587 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 10641b2c0a69,42587,1730976488749 2024-11-07T10:48:09,231 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,42587,1730976488749' 2024-11-07T10:48:09,231 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T10:48:09,231 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T10:48:09,232 DEBUG [RS:0;10641b2c0a69:42587 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T10:48:09,232 INFO [RS:0;10641b2c0a69:42587 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T10:48:09,232 INFO [RS:0;10641b2c0a69:42587 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T10:48:09,266 WARN [10641b2c0a69:42249 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-07T10:48:09,324 INFO [RS:1;10641b2c0a69:32933 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:48:09,325 INFO [RS:1;10641b2c0a69:32933 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=10641b2c0a69%2C32933%2C1730976488787, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,32933,1730976488787, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs, maxLogs=55 2024-11-07T10:48:09,333 INFO [RS:0;10641b2c0a69:42587 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:48:09,333 INFO [RS:0;10641b2c0a69:42587 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=10641b2c0a69%2C42587%2C1730976488749, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,42587,1730976488749, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs, maxLogs=55 2024-11-07T10:48:09,344 INFO [RS:1;10641b2c0a69:32933 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,32933,1730976488787/10641b2c0a69%2C32933%2C1730976488787.1730976489326 2024-11-07T10:48:09,344 DEBUG [RS:1;10641b2c0a69:32933 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T10:48:09,353 INFO [RS:0;10641b2c0a69:42587 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,42587,1730976488749/10641b2c0a69%2C42587%2C1730976488749.1730976489335 2024-11-07T10:48:09,353 DEBUG [RS:0;10641b2c0a69:42587 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T10:48:09,517 DEBUG [10641b2c0a69:42249 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-07T10:48:09,517 DEBUG [10641b2c0a69:42249 {}] balancer.BalancerClusterState(204): Hosts are {10641b2c0a69=0} racks are {/default-rack=0} 2024-11-07T10:48:09,518 DEBUG [10641b2c0a69:42249 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T10:48:09,518 DEBUG [10641b2c0a69:42249 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T10:48:09,518 DEBUG [10641b2c0a69:42249 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T10:48:09,518 DEBUG [10641b2c0a69:42249 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T10:48:09,518 INFO [10641b2c0a69:42249 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T10:48:09,518 INFO [10641b2c0a69:42249 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T10:48:09,518 DEBUG [10641b2c0a69:42249 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T10:48:09,519 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=10641b2c0a69,32933,1730976488787 2024-11-07T10:48:09,520 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,32933,1730976488787, state=OPENING 2024-11-07T10:48:09,522 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T10:48:09,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:09,524 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:48:09,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:09,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,32933,1730976488787}] 2024-11-07T10:48:09,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:09,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:09,677 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T10:48:09,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46359, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T10:48:09,681 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T10:48:09,681 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:48:09,682 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T10:48:09,682 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=10641b2c0a69%2C32933%2C1730976488787.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,32933,1730976488787, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs, maxLogs=55 2024-11-07T10:48:09,700 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,32933,1730976488787/10641b2c0a69%2C32933%2C1730976488787.meta.1730976489683.meta 2024-11-07T10:48:09,701 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T10:48:09,701 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:48:09,702 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T10:48:09,702 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T10:48:09,702 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T10:48:09,702 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T10:48:09,702 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:09,702 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T10:48:09,703 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T10:48:09,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-07T10:48:09,705 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T10:48:09,705 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:09,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-07T10:48:09,706 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T10:48:09,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:09,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-07T10:48:09,707 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T10:48:09,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:09,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-07T10:48:09,709 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T10:48:09,709 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:48:09,709 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T10:48:09,710 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740 2024-11-07T10:48:09,710 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740 2024-11-07T10:48:09,711 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T10:48:09,711 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T10:48:09,711 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:48:09,712 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T10:48:09,712 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75471720, jitterRate=0.12461626529693604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:48:09,712 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T10:48:09,713 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730976489703Writing region info on filesystem at 1730976489703Initializing all the Stores at 1730976489703Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976489703Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976489703Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976489703Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976489703Cleaning up temporary data from old regions at 1730976489711 (+8 ms)Running coprocessor post-open hooks at 1730976489712 (+1 ms)Region opened successfully at 1730976489713 (+1 ms) 2024-11-07T10:48:09,713 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730976489677 2024-11-07T10:48:09,716 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T10:48:09,716 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T10:48:09,717 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,32933,1730976488787 2024-11-07T10:48:09,717 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,32933,1730976488787, state=OPEN 2024-11-07T10:48:09,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:48:09,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:48:09,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:48:09,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:09,720 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=10641b2c0a69,32933,1730976488787 2024-11-07T10:48:09,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:09,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:48:09,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T10:48:09,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,32933,1730976488787 in 196 msec 2024-11-07T10:48:09,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T10:48:09,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-11-07T10:48:09,724 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T10:48:09,724 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T10:48:09,727 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:48:09,727 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,32933,1730976488787, seqNum=-1] 2024-11-07T10:48:09,728 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:48:09,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36313, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:48:09,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 693 msec 2024-11-07T10:48:09,734 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730976489734, completionTime=-1 2024-11-07T10:48:09,734 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-07T10:48:09,734 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:48:09,736 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T10:48:09,736 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730976549736 2024-11-07T10:48:09,736 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730976609736 2024-11-07T10:48:09,737 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-07T10:48:09,737 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,42249,1730976488690-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,737 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,42249,1730976488690-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,737 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,42249,1730976488690-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,737 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-10641b2c0a69:42249, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,737 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,737 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,740 DEBUG [master/10641b2c0a69:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T10:48:09,742 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.908sec 2024-11-07T10:48:09,743 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T10:48:09,743 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T10:48:09,743 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T10:48:09,743 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T10:48:09,743 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T10:48:09,743 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,42249,1730976488690-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:48:09,743 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,42249,1730976488690-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T10:48:09,744 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T10:48:09,744 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T10:48:09,744 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,42249,1730976488690-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:09,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T10:48:09,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 10641b2c0a69,42249,-1 for getting cluster id 2024-11-07T10:48:09,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T10:48:09,817 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75843e14-39e7-4ded-9e6c-322c5bc461ae' 2024-11-07T10:48:09,818 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T10:48:09,818 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75843e14-39e7-4ded-9e6c-322c5bc461ae" 2024-11-07T10:48:09,818 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55186dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T10:48:09,818 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [10641b2c0a69,42249,-1] 2024-11-07T10:48:09,818 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T10:48:09,819 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:09,820 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T10:48:09,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T10:48:09,822 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:48:09,824 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,32933,1730976488787, seqNum=-1] 2024-11-07T10:48:09,824 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:48:09,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:48:09,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=10641b2c0a69,42249,1730976488690 2024-11-07T10:48:09,829 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T10:48:09,830 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 10641b2c0a69,42249,1730976488690 2024-11-07T10:48:09,830 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@37c9a8c4 2024-11-07T10:48:09,830 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T10:48:09,832 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T10:48:09,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42249 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1730976489829', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T10:48:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42249 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730976489829 2024-11-07T10:48:09,835 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730976489829 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T10:48:09,835 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:09,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42249 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1730976489829" procId is: 4 2024-11-07T10:48:09,836 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730976489829 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T10:48:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:48:09,847 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730976489829-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0076bb6b2c101b8f73afa34b8c46fcb0, NAME => 'testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1730976489829', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 2024-11-07T10:48:09,853 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730976489829-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:09,853 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730976489829-pool-0 {}] regionserver.HRegion(1722): Closing 0076bb6b2c101b8f73afa34b8c46fcb0, disabling compactions & flushes 2024-11-07T10:48:09,853 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730976489829-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:09,853 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730976489829-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:09,853 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730976489829-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. after waiting 0 ms 2024-11-07T10:48:09,854 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730976489829-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:09,854 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730976489829-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:09,854 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730976489829-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0076bb6b2c101b8f73afa34b8c46fcb0: Waiting for close lock at 1730976489853Disabling compacts and flushes for region at 1730976489853Disabling writes for close at 1730976489854 (+1 ms)Writing region close event to WAL at 1730976489854Closed at 1730976489854 2024-11-07T10:48:09,855 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730976489829 execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T10:48:09,855 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1730976489855"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976489855"}]},"ts":"1730976489855"} 2024-11-07T10:48:09,861 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T10:48:09,862 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730976489829 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T10:48:09,862 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1730976489829","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730976489862"}]},"ts":"1730976489862"} 2024-11-07T10:48:09,864 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1730976489829, state=ENABLING in hbase:meta 2024-11-07T10:48:09,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {10641b2c0a69=0} racks are {/default-rack=0} 2024-11-07T10:48:09,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T10:48:09,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T10:48:09,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T10:48:09,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T10:48:09,865 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T10:48:09,865 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T10:48:09,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T10:48:09,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN}] 2024-11-07T10:48:09,867 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN 2024-11-07T10:48:09,868 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN; state=OFFLINE, location=10641b2c0a69,42587,1730976488749; forceNewPlan=false, retain=false 2024-11-07T10:48:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:48:10,018 INFO [10641b2c0a69:42249 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-07T10:48:10,019 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0076bb6b2c101b8f73afa34b8c46fcb0, regionState=OPENING, regionLocation=10641b2c0a69,42587,1730976488749 2024-11-07T10:48:10,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN because future has completed 2024-11-07T10:48:10,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0076bb6b2c101b8f73afa34b8c46fcb0, server=10641b2c0a69,42587,1730976488749}] 2024-11-07T10:48:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:48:10,174 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T10:48:10,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42867, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T10:48:10,178 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:10,179 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0076bb6b2c101b8f73afa34b8c46fcb0, NAME => 'testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0.', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:48:10,179 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1730976489829 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,179 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:10,179 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,179 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,180 INFO [StoreOpener-0076bb6b2c101b8f73afa34b8c46fcb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,184 INFO [StoreOpener-0076bb6b2c101b8f73afa34b8c46fcb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0076bb6b2c101b8f73afa34b8c46fcb0 columnFamilyName family 2024-11-07T10:48:10,184 DEBUG [StoreOpener-0076bb6b2c101b8f73afa34b8c46fcb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:10,185 INFO [StoreOpener-0076bb6b2c101b8f73afa34b8c46fcb0-1 {}] regionserver.HStore(327): Store=0076bb6b2c101b8f73afa34b8c46fcb0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:10,185 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,186 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,186 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,186 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,186 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,186 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,192 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:48:10,193 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0076bb6b2c101b8f73afa34b8c46fcb0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67277844, jitterRate=0.002517998218536377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T10:48:10,193 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:10,194 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0076bb6b2c101b8f73afa34b8c46fcb0: Running coprocessor pre-open hook at 1730976490180Writing region info on filesystem at 1730976490180Initializing all the Stores at 1730976490180Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976490180Cleaning up temporary data from old regions at 1730976490186 (+6 ms)Running coprocessor post-open hooks at 1730976490193 (+7 ms)Region opened successfully at 1730976490193 2024-11-07T10:48:10,194 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0., pid=6, masterSystemTime=1730976490174 2024-11-07T10:48:10,196 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:10,197 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:10,197 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0076bb6b2c101b8f73afa34b8c46fcb0, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,42587,1730976488749 2024-11-07T10:48:10,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0076bb6b2c101b8f73afa34b8c46fcb0, server=10641b2c0a69,42587,1730976488749 because future has completed 2024-11-07T10:48:10,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T10:48:10,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0076bb6b2c101b8f73afa34b8c46fcb0, server=10641b2c0a69,42587,1730976488749 in 179 msec 2024-11-07T10:48:10,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T10:48:10,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN in 336 msec 2024-11-07T10:48:10,205 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730976489829 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T10:48:10,205 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1730976489829","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730976490205"}]},"ts":"1730976490205"} 2024-11-07T10:48:10,207 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1730976489829, state=ENABLED in hbase:meta 2024-11-07T10:48:10,208 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730976489829 execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T10:48:10,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730976489829 in 375 msec 2024-11-07T10:48:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:48:10,468 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1730976489829 completed 2024-11-07T10:48:10,468 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1730976489829 get assigned. Timeout = 60000ms 2024-11-07T10:48:10,468 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T10:48:10,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1730976489829 assigned to meta. Checking AM states. 2024-11-07T10:48:10,472 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T10:48:10,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1730976489829 assigned. 2024-11-07T10:48:10,476 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1730976489829', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0., hostname=10641b2c0a69,42587,1730976488749, seqNum=2] 2024-11-07T10:48:10,477 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:48:10,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:48:10,483 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0076bb6b2c101b8f73afa34b8c46fcb0 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-07T10:48:10,498 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f3b7201f223a41d48d67677ee65a5a08 2024-11-07T10:48:10,517 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/.tmp/family/f3b7201f223a41d48d67677ee65a5a08 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-07T10:48:10,518 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/.tmp/family/f3b7201f223a41d48d67677ee65a5a08 2024-11-07T10:48:10,522 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/.tmp/family/f3b7201f223a41d48d67677ee65a5a08 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08 2024-11-07T10:48:10,523 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08, delay=939 ms 2024-11-07T10:48:10,524 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08, entries=2, sequenceid=6, filesize=4.9 K 2024-11-07T10:48:10,525 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 0076bb6b2c101b8f73afa34b8c46fcb0 in 42ms, sequenceid=6, compaction requested=false 2024-11-07T10:48:10,525 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1730976489829' 2024-11-07T10:48:10,526 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0076bb6b2c101b8f73afa34b8c46fcb0: 2024-11-07T10:48:10,939 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T10:48:11,463 DEBUG [hfile-prefetch-1730976488123 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f3b7201f223a41d48d67677ee65a5a08_0. 2024-11-07T10:48:11,464 DEBUG [hfile-prefetch-1730976488123 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f3b7201f223a41d48d67677ee65a5a08_117. 2024-11-07T10:48:11,464 DEBUG [hfile-prefetch-1730976488123 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08, with total blocks 2, and data blocks 1 2024-11-07T10:48:11,465 DEBUG [hfile-prefetch-1730976488123 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-07T10:48:11,465 DEBUG [hfile-prefetch-1730976488123 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-07T10:48:11,465 DEBUG [hfile-prefetch-1730976488123 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08. Will sleep 100ms and try the verification again. 2024-11-07T10:48:11,498 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-07T10:48:11,498 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-07T10:48:11,498 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T10:48:11,498 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T10:48:11,502 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T10:48:11,502 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730976491498 2024-11-07T10:48:11,502 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-07T10:48:11,526 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;10641b2c0a69:42587,5,FailOnTimeoutGroup] 2024-11-07T10:48:11,526 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '10641b2c0a69,42587,1730976488749' ***** 2024-11-07T10:48:11,526 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-07T10:48:11,527 INFO [RS:0;10641b2c0a69:42587 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T10:48:11,527 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T10:48:11,527 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T10:48:11,527 INFO [RS:0;10641b2c0a69:42587 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.persistence 2024-11-07T10:48:11,527 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T10:48:11,527 INFO [RS:0;10641b2c0a69:42587 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T10:48:11,527 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T10:48:11,537 DEBUG [RS:0;10641b2c0a69:42587 {}] bucket.BucketCache(1488): Thread RS:0;10641b2c0a69:42587 started persisting bucket cache to file 2024-11-07T10:48:11,537 DEBUG [RS:0;10641b2c0a69:42587 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T10:48:11,537 DEBUG [RS:0;10641b2c0a69:42587 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T10:48:11,540 DEBUG [RS:0;10641b2c0a69:42587 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-07T10:48:11,540 DEBUG [RS:0;10641b2c0a69:42587 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1730976491537 2024-11-07T10:48:11,540 DEBUG [RS:0;10641b2c0a69:42587 {}] bucket.BucketCache(1510): Thread RS:0;10641b2c0a69:42587 finished persisting bucket cache to file, renaming 2024-11-07T10:48:11,541 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T10:48:11,541 INFO [RS:0;10641b2c0a69:42587 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T10:48:11,541 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T10:48:11,541 INFO [RS:0;10641b2c0a69:42587 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T10:48:11,541 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(3091): Received CLOSE for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:11,541 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(959): stopping server 10641b2c0a69,42587,1730976488749 2024-11-07T10:48:11,541 INFO [RS:0;10641b2c0a69:42587 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:48:11,541 INFO [RS:0;10641b2c0a69:42587 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;10641b2c0a69:42587. 2024-11-07T10:48:11,541 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T10:48:11,542 DEBUG [RS:0;10641b2c0a69:42587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:48:11,542 DEBUG [RS:0;10641b2c0a69:42587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:11,542 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0076bb6b2c101b8f73afa34b8c46fcb0, disabling compactions & flushes 2024-11-07T10:48:11,542 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:11,542 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T10:48:11,542 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:11,542 DEBUG [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(1325): Online Regions={0076bb6b2c101b8f73afa34b8c46fcb0=testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0.} 2024-11-07T10:48:11,542 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. after waiting 0 ms 2024-11-07T10:48:11,542 DEBUG [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(1351): Waiting on 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:11,542 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:11,542 DEBUG [StoreFileCloser-0076bb6b2c101b8f73afa34b8c46fcb0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08 2024-11-07T10:48:11,544 WARN [hfile-prefetch-1730976488123 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-07T10:48:11,545 DEBUG [hfile-prefetch-1730976488123 {}] hfile.PrefetchExecutor(121): Prefetch completed for f3b7201f223a41d48d67677ee65a5a08 2024-11-07T10:48:11,548 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-07T10:48:11,548 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:11,548 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0076bb6b2c101b8f73afa34b8c46fcb0: Waiting for close lock at 1730976491541Running coprocessor pre-close hooks at 1730976491542 (+1 ms)Disabling compacts and flushes for region at 1730976491542Disabling writes for close at 1730976491542Writing region close event to WAL at 1730976491543 (+1 ms)Running coprocessor post-close hooks at 1730976491548 (+5 ms)Closed at 1730976491548 2024-11-07T10:48:11,548 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:11,585 INFO [regionserver/10641b2c0a69:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:11,742 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(976): stopping server 10641b2c0a69,42587,1730976488749; all regions closed. 2024-11-07T10:48:11,744 DEBUG [RS:0;10641b2c0a69:42587 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs 2024-11-07T10:48:11,744 INFO [RS:0;10641b2c0a69:42587 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C42587%2C1730976488749:(num 1730976489335) 2024-11-07T10:48:11,744 DEBUG [RS:0;10641b2c0a69:42587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:11,744 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:11,744 INFO [RS:0;10641b2c0a69:42587 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:48:11,744 INFO [RS:0;10641b2c0a69:42587 {}] hbase.ChoreService(370): Chore service for: regionserver/10641b2c0a69:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T10:48:11,745 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T10:48:11,745 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T10:48:11,745 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T10:48:11,745 INFO [RS:0;10641b2c0a69:42587 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:48:11,745 INFO [regionserver/10641b2c0a69:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:48:11,745 INFO [RS:0;10641b2c0a69:42587 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42587 2024-11-07T10:48:11,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/10641b2c0a69,42587,1730976488749 2024-11-07T10:48:11,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:48:11,750 INFO [RS:0;10641b2c0a69:42587 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:48:11,751 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [10641b2c0a69,42587,1730976488749] 2024-11-07T10:48:11,753 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/10641b2c0a69,42587,1730976488749 already deleted, retry=false 2024-11-07T10:48:11,753 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 10641b2c0a69,42587,1730976488749 on 10641b2c0a69,42249,1730976488690 2024-11-07T10:48:11,757 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 10641b2c0a69,42587,1730976488749, splitWal=true, meta=false 2024-11-07T10:48:11,760 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 10641b2c0a69,42587,1730976488749 (carryingMeta=false) 10641b2c0a69,42587,1730976488749/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@d63181e[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-07T10:48:11,760 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 10641b2c0a69,42587,1730976488749, splitWal=true, meta=false 2024-11-07T10:48:11,761 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 10641b2c0a69,42587,1730976488749 had 1 regions 2024-11-07T10:48:11,762 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,42587,1730976488749, splitWal=true, meta=false, isMeta: false 2024-11-07T10:48:11,763 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,42587,1730976488749-splitting 2024-11-07T10:48:11,763 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,42587,1730976488749-splitting dir is empty, no logs to split. 2024-11-07T10:48:11,763 INFO [PEWorker-2 {}] master.SplitWALManager(105): 10641b2c0a69,42587,1730976488749 WAL count=0, meta=false 2024-11-07T10:48:11,765 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,42587,1730976488749-splitting dir is empty, no logs to split. 2024-11-07T10:48:11,765 INFO [PEWorker-2 {}] master.SplitWALManager(105): 10641b2c0a69,42587,1730976488749 WAL count=0, meta=false 2024-11-07T10:48:11,765 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 10641b2c0a69,42587,1730976488749 WAL splitting is done? wals=0, meta=false 2024-11-07T10:48:11,766 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 10641b2c0a69,42587,1730976488749 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,42587,1730976488749-splitting does not exist 2024-11-07T10:48:11,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN}] 2024-11-07T10:48:11,768 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN 2024-11-07T10:48:11,769 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:48:11,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:11,852 INFO [RS:0;10641b2c0a69:42587 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:48:11,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42587-0x10019db27800001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:11,852 INFO [RS:0;10641b2c0a69:42587 {}] regionserver.HRegionServer(1031): Exiting; stopping=10641b2c0a69,42587,1730976488749; zookeeper connection closed. 2024-11-07T10:48:11,852 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3085e653 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3085e653 2024-11-07T10:48:11,920 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0076bb6b2c101b8f73afa34b8c46fcb0, regionState=OPENING, regionLocation=10641b2c0a69,32933,1730976488787 2024-11-07T10:48:11,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN because future has completed 2024-11-07T10:48:11,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0076bb6b2c101b8f73afa34b8c46fcb0, server=10641b2c0a69,32933,1730976488787}] 2024-11-07T10:48:12,043 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:48:12,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:12,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:12,043 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:48:12,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:48:12,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:48:12,043 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T10:48:12,043 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:48:12,044 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45325 2024-11-07T10:48:12,045 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45325 connecting to ZooKeeper ensemble=127.0.0.1:53542 2024-11-07T10:48:12,046 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:12,046 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:12,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453250x0, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:48:12,050 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45325-0x10019db27800003 connected 2024-11-07T10:48:12,050 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-07T10:48:12,050 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-07T10:48:12,051 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T10:48:12,052 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.cache 2024-11-07T10:48:12,052 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T10:48:12,052 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T10:48:12,052 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T10:48:12,052 INFO [Thread-200 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T10:48:12,053 INFO [Thread-200 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-07T10:48:12,055 INFO [Thread-200 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-07T10:48:12,055 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T10:48:12,057 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T10:48:12,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T10:48:12,058 DEBUG [Thread-200 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-07T10:48:12,059 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T10:48:12,059 INFO [Thread-200 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-07T10:48:12,059 INFO [Thread-200 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.persistence is complete. 2024-11-07T10:48:12,059 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:48:12,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45325 2024-11-07T10:48:12,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45325 2024-11-07T10:48:12,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45325 2024-11-07T10:48:12,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45325 2024-11-07T10:48:12,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45325 2024-11-07T10:48:12,066 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(746): ClusterId : 75843e14-39e7-4ded-9e6c-322c5bc461ae 2024-11-07T10:48:12,066 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T10:48:12,069 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T10:48:12,069 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T10:48:12,072 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T10:48:12,072 DEBUG [RS:2;10641b2c0a69:45325 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22db2749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:48:12,080 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:12,080 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0076bb6b2c101b8f73afa34b8c46fcb0, NAME => 'testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0.', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:48:12,081 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1730976489829 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,081 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:48:12,081 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,081 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,082 INFO [StoreOpener-0076bb6b2c101b8f73afa34b8c46fcb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,083 INFO [StoreOpener-0076bb6b2c101b8f73afa34b8c46fcb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0076bb6b2c101b8f73afa34b8c46fcb0 columnFamilyName family 2024-11-07T10:48:12,083 DEBUG [StoreOpener-0076bb6b2c101b8f73afa34b8c46fcb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:48:12,085 DEBUG [StoreFileOpener-0076bb6b2c101b8f73afa34b8c46fcb0-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08, delay=993 ms 2024-11-07T10:48:12,085 DEBUG [RS:2;10641b2c0a69:45325 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;10641b2c0a69:45325 2024-11-07T10:48:12,085 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T10:48:12,085 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T10:48:12,085 DEBUG [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T10:48:12,085 DEBUG [StoreOpener-0076bb6b2c101b8f73afa34b8c46fcb0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08 2024-11-07T10:48:12,086 INFO [StoreOpener-0076bb6b2c101b8f73afa34b8c46fcb0-1 {}] regionserver.HStore(327): Store=0076bb6b2c101b8f73afa34b8c46fcb0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:48:12,086 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,086 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,42249,1730976488690 with port=45325, startcode=1730976492042 2024-11-07T10:48:12,087 DEBUG [RS:2;10641b2c0a69:45325 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T10:48:12,087 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,087 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,088 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,088 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,089 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49177, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T10:48:12,089 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,090 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42249 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 10641b2c0a69,45325,1730976492042 2024-11-07T10:48:12,089 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 0076bb6b2c101b8f73afa34b8c46fcb0; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70472477, jitterRate=0.0501217395067215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T10:48:12,090 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,090 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42249 {}] master.ServerManager(517): Registering regionserver=10641b2c0a69,45325,1730976492042 2024-11-07T10:48:12,091 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0076bb6b2c101b8f73afa34b8c46fcb0: Running coprocessor pre-open hook at 1730976492081Writing region info on filesystem at 1730976492081Initializing all the Stores at 1730976492082 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976492082Cleaning up temporary data from old regions at 1730976492088 (+6 ms)Running coprocessor post-open hooks at 1730976492090 (+2 ms)Region opened successfully at 1730976492091 (+1 ms) 2024-11-07T10:48:12,091 DEBUG [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9 2024-11-07T10:48:12,091 DEBUG [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T10:48:12,091 DEBUG [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T10:48:12,092 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0., pid=9, masterSystemTime=1730976492075 2024-11-07T10:48:12,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:48:12,094 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:12,094 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:12,094 DEBUG [RS:2;10641b2c0a69:45325 {}] zookeeper.ZKUtil(111): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/10641b2c0a69,45325,1730976492042 2024-11-07T10:48:12,094 WARN [RS:2;10641b2c0a69:45325 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:48:12,094 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [10641b2c0a69,45325,1730976492042] 2024-11-07T10:48:12,095 INFO [RS:2;10641b2c0a69:45325 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:48:12,095 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0076bb6b2c101b8f73afa34b8c46fcb0, regionState=OPEN, openSeqNum=10, regionLocation=10641b2c0a69,32933,1730976488787 2024-11-07T10:48:12,095 DEBUG [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,45325,1730976492042 2024-11-07T10:48:12,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0076bb6b2c101b8f73afa34b8c46fcb0, server=10641b2c0a69,32933,1730976488787 because future has completed 2024-11-07T10:48:12,103 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T10:48:12,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-07T10:48:12,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 0076bb6b2c101b8f73afa34b8c46fcb0, server=10641b2c0a69,32933,1730976488787 in 178 msec 2024-11-07T10:48:12,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T10:48:12,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730976489829, region=0076bb6b2c101b8f73afa34b8c46fcb0, ASSIGN in 337 msec 2024-11-07T10:48:12,106 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 10641b2c0a69,42587,1730976488749 after splitting done 2024-11-07T10:48:12,106 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T10:48:12,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 10641b2c0a69,42587,1730976488749, splitWal=true, meta=false in 352 msec 2024-11-07T10:48:12,109 INFO [RS:2;10641b2c0a69:45325 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T10:48:12,109 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,109 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T10:48:12,109 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T10:48:12,109 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:12,110 DEBUG [RS:2;10641b2c0a69:45325 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:48:12,112 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,113 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,113 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,113 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,113 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,113 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,45325,1730976492042-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:48:12,129 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T10:48:12,129 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,45325,1730976492042-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,130 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,130 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.Replication(171): 10641b2c0a69,45325,1730976492042 started 2024-11-07T10:48:12,145 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:48:12,145 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(1482): Serving as 10641b2c0a69,45325,1730976492042, RpcServer on 10641b2c0a69/172.17.0.2:45325, sessionid=0x10019db27800003 2024-11-07T10:48:12,145 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T10:48:12,145 DEBUG [RS:2;10641b2c0a69:45325 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 10641b2c0a69,45325,1730976492042 2024-11-07T10:48:12,145 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,45325,1730976492042' 2024-11-07T10:48:12,145 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T10:48:12,146 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T10:48:12,147 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T10:48:12,147 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T10:48:12,147 DEBUG [RS:2;10641b2c0a69:45325 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 10641b2c0a69,45325,1730976492042 2024-11-07T10:48:12,147 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,45325,1730976492042' 2024-11-07T10:48:12,147 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T10:48:12,147 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T10:48:12,147 DEBUG [RS:2;10641b2c0a69:45325 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T10:48:12,148 INFO [RS:2;10641b2c0a69:45325 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T10:48:12,148 INFO [RS:2;10641b2c0a69:45325 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T10:48:12,248 INFO [RS:2;10641b2c0a69:45325 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:48:12,249 INFO [RS:2;10641b2c0a69:45325 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=10641b2c0a69%2C45325%2C1730976492042, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,45325,1730976492042, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs, maxLogs=55 2024-11-07T10:48:12,268 INFO [RS:2;10641b2c0a69:45325 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/WALs/10641b2c0a69,45325,1730976492042/10641b2c0a69%2C45325%2C1730976492042.1730976492250 2024-11-07T10:48:12,269 DEBUG [RS:2;10641b2c0a69:45325 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T10:48:12,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T10:48:12,646 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T10:48:12,646 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:48:12,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:12,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:12,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T10:48:12,646 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T10:48:12,646 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=249403659, stopped=false 2024-11-07T10:48:12,647 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=10641b2c0a69,42249,1730976488690 2024-11-07T10:48:12,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:12,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:12,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T10:48:12,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:12,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:12,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:12,649 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:48:12,649 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T10:48:12,649 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:48:12,650 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:12,650 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:12,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:12,650 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '10641b2c0a69,32933,1730976488787' ***** 2024-11-07T10:48:12,650 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:48:12,650 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T10:48:12,650 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '10641b2c0a69,45325,1730976492042' ***** 2024-11-07T10:48:12,650 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T10:48:12,650 INFO [RS:1;10641b2c0a69:32933 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T10:48:12,650 INFO [RS:1;10641b2c0a69:32933 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.persistence 2024-11-07T10:48:12,650 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T10:48:12,650 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T10:48:12,650 INFO [RS:2;10641b2c0a69:45325 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T10:48:12,650 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T10:48:12,651 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T10:48:12,651 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T10:48:12,651 INFO [RS:1;10641b2c0a69:32933 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T10:48:12,651 INFO [RS:2;10641b2c0a69:45325 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bbb97246-b7d3-2e78-82b0-d2658e4cf7b6/bucket.persistence 2024-11-07T10:48:12,651 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T10:48:12,651 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T10:48:12,651 INFO [RS:2;10641b2c0a69:45325 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T10:48:12,651 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T10:48:12,661 DEBUG [RS:1;10641b2c0a69:32933 {}] bucket.BucketCache(1488): Thread RS:1;10641b2c0a69:32933 started persisting bucket cache to file 2024-11-07T10:48:12,661 DEBUG [RS:1;10641b2c0a69:32933 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T10:48:12,661 DEBUG [RS:1;10641b2c0a69:32933 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T10:48:12,661 DEBUG [RS:2;10641b2c0a69:45325 {}] bucket.BucketCache(1488): Thread RS:2;10641b2c0a69:45325 started persisting bucket cache to file 2024-11-07T10:48:12,662 DEBUG [RS:2;10641b2c0a69:45325 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T10:48:12,662 DEBUG [RS:2;10641b2c0a69:45325 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T10:48:12,665 DEBUG [RS:1;10641b2c0a69:32933 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T10:48:12,665 DEBUG [RS:1;10641b2c0a69:32933 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730976492661 2024-11-07T10:48:12,665 DEBUG [RS:1;10641b2c0a69:32933 {}] bucket.BucketCache(1510): Thread RS:1;10641b2c0a69:32933 finished persisting bucket cache to file, renaming 2024-11-07T10:48:12,665 DEBUG [RS:2;10641b2c0a69:45325 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-07T10:48:12,665 DEBUG [RS:2;10641b2c0a69:45325 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1730976492662 2024-11-07T10:48:12,665 DEBUG [RS:2;10641b2c0a69:45325 {}] bucket.BucketCache(1510): Thread RS:2;10641b2c0a69:45325 finished persisting bucket cache to file, renaming 2024-11-07T10:48:12,665 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T10:48:12,666 INFO [RS:1;10641b2c0a69:32933 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T10:48:12,666 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T10:48:12,666 INFO [RS:1;10641b2c0a69:32933 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T10:48:12,666 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(3091): Received CLOSE for 0076bb6b2c101b8f73afa34b8c46fcb0 2024-11-07T10:48:12,666 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(959): stopping server 10641b2c0a69,32933,1730976488787 2024-11-07T10:48:12,666 INFO [RS:1;10641b2c0a69:32933 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:48:12,666 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T10:48:12,666 INFO [RS:1;10641b2c0a69:32933 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;10641b2c0a69:32933. 2024-11-07T10:48:12,666 DEBUG [RS:1;10641b2c0a69:32933 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:48:12,666 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T10:48:12,666 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T10:48:12,666 INFO [RS:2;10641b2c0a69:45325 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T10:48:12,666 DEBUG [RS:1;10641b2c0a69:32933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:12,666 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0076bb6b2c101b8f73afa34b8c46fcb0, disabling compactions & flushes 2024-11-07T10:48:12,666 INFO [RS:2;10641b2c0a69:45325 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T10:48:12,667 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(959): stopping server 10641b2c0a69,45325,1730976492042 2024-11-07T10:48:12,667 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:12,667 INFO [RS:2;10641b2c0a69:45325 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:48:12,667 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T10:48:12,667 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:12,667 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T10:48:12,667 INFO [RS:2;10641b2c0a69:45325 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;10641b2c0a69:45325. 2024-11-07T10:48:12,667 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T10:48:12,667 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. after waiting 0 ms 2024-11-07T10:48:12,667 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:12,667 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T10:48:12,667 DEBUG [RS:2;10641b2c0a69:45325 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:48:12,667 DEBUG [RS:2;10641b2c0a69:45325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:12,667 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(976): stopping server 10641b2c0a69,45325,1730976492042; all regions closed. 2024-11-07T10:48:12,668 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-07T10:48:12,668 DEBUG [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 0076bb6b2c101b8f73afa34b8c46fcb0=testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0.} 2024-11-07T10:48:12,668 DEBUG [StoreFileCloser-0076bb6b2c101b8f73afa34b8c46fcb0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/family/f3b7201f223a41d48d67677ee65a5a08 2024-11-07T10:48:12,668 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T10:48:12,668 DEBUG [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(1351): Waiting on 0076bb6b2c101b8f73afa34b8c46fcb0, 1588230740 2024-11-07T10:48:12,668 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T10:48:12,668 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T10:48:12,668 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T10:48:12,669 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 1 ms 2024-11-07T10:48:12,669 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T10:48:12,669 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-07T10:48:12,670 DEBUG [RS:2;10641b2c0a69:45325 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs 2024-11-07T10:48:12,670 INFO [RS:2;10641b2c0a69:45325 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C45325%2C1730976492042:(num 1730976492250) 2024-11-07T10:48:12,670 DEBUG [RS:2;10641b2c0a69:45325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:12,671 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:12,671 INFO [RS:2;10641b2c0a69:45325 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:48:12,671 INFO [RS:2;10641b2c0a69:45325 {}] hbase.ChoreService(370): Chore service for: regionserver/10641b2c0a69:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T10:48:12,671 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T10:48:12,671 INFO [regionserver/10641b2c0a69:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:48:12,671 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T10:48:12,671 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T10:48:12,671 INFO [RS:2;10641b2c0a69:45325 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:48:12,672 INFO [RS:2;10641b2c0a69:45325 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45325 2024-11-07T10:48:12,673 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/default/testBlockEvictionOnGracefulStop1730976489829/0076bb6b2c101b8f73afa34b8c46fcb0/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-07T10:48:12,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/10641b2c0a69,45325,1730976492042 2024-11-07T10:48:12,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:48:12,674 INFO [RS:2;10641b2c0a69:45325 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:48:12,674 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:12,674 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0076bb6b2c101b8f73afa34b8c46fcb0: Waiting for close lock at 1730976492666Running coprocessor pre-close hooks at 1730976492666Disabling compacts and flushes for region at 1730976492666Disabling writes for close at 1730976492667 (+1 ms)Writing region close event to WAL at 1730976492669 (+2 ms)Running coprocessor post-close hooks at 1730976492674 (+5 ms)Closed at 1730976492674 2024-11-07T10:48:12,674 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0. 2024-11-07T10:48:12,675 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [10641b2c0a69,45325,1730976492042] 2024-11-07T10:48:12,676 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/10641b2c0a69,45325,1730976492042 already deleted, retry=false 2024-11-07T10:48:12,677 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 10641b2c0a69,45325,1730976492042 expired; onlineServers=1 2024-11-07T10:48:12,680 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 602f331985d049e6b78490b10fce3c03 2024-11-07T10:48:12,685 INFO [regionserver/10641b2c0a69:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:12,694 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/info/602f331985d049e6b78490b10fce3c03 is 209, key is testBlockEvictionOnGracefulStop1730976489829,,1730976489833.0076bb6b2c101b8f73afa34b8c46fcb0./info:regioninfo/1730976492095/Put/seqid=0 2024-11-07T10:48:12,695 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/info/602f331985d049e6b78490b10fce3c03 2024-11-07T10:48:12,703 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d1884697d5364f75b42683e097bdacc6 2024-11-07T10:48:12,713 INFO [regionserver/10641b2c0a69:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:12,716 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/ns/d1884697d5364f75b42683e097bdacc6 is 43, key is default/ns:d/1730976489730/Put/seqid=0 2024-11-07T10:48:12,717 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/ns/d1884697d5364f75b42683e097bdacc6 2024-11-07T10:48:12,725 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b3b4abb5d7404a208cb2d7ceeeb5d0a4 2024-11-07T10:48:12,739 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/table/b3b4abb5d7404a208cb2d7ceeeb5d0a4 is 80, key is testBlockEvictionOnGracefulStop1730976489829/table:state/1730976490205/Put/seqid=0 2024-11-07T10:48:12,739 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/table/b3b4abb5d7404a208cb2d7ceeeb5d0a4 2024-11-07T10:48:12,741 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/info/602f331985d049e6b78490b10fce3c03 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/info/602f331985d049e6b78490b10fce3c03 2024-11-07T10:48:12,743 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/info/602f331985d049e6b78490b10fce3c03, delay=903 ms 2024-11-07T10:48:12,744 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/info/602f331985d049e6b78490b10fce3c03, entries=14, sequenceid=13, filesize=7.5 K 2024-11-07T10:48:12,744 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/ns/d1884697d5364f75b42683e097bdacc6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/ns/d1884697d5364f75b42683e097bdacc6 2024-11-07T10:48:12,745 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/ns/d1884697d5364f75b42683e097bdacc6, delay=960 ms 2024-11-07T10:48:12,746 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/ns/d1884697d5364f75b42683e097bdacc6, entries=2, sequenceid=13, filesize=5.0 K 2024-11-07T10:48:12,747 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/.tmp/table/b3b4abb5d7404a208cb2d7ceeeb5d0a4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/table/b3b4abb5d7404a208cb2d7ceeeb5d0a4 2024-11-07T10:48:12,748 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/table/b3b4abb5d7404a208cb2d7ceeeb5d0a4, delay=945 ms 2024-11-07T10:48:12,749 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/table/b3b4abb5d7404a208cb2d7ceeeb5d0a4, entries=2, sequenceid=13, filesize=5.3 K 2024-11-07T10:48:12,749 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=13, compaction requested=false 2024-11-07T10:48:12,750 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/info/602f331985d049e6b78490b10fce3c03 2024-11-07T10:48:12,751 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/ns/d1884697d5364f75b42683e097bdacc6 2024-11-07T10:48:12,751 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/table/b3b4abb5d7404a208cb2d7ceeeb5d0a4 2024-11-07T10:48:12,755 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-07T10:48:12,756 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T10:48:12,756 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T10:48:12,756 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730976492668Running coprocessor pre-close hooks at 1730976492668Disabling compacts and flushes for region at 1730976492668Disabling writes for close at 1730976492669 (+1 ms)Obtaining lock to block concurrent updates at 1730976492669Preparing flush snapshotting stores in 1588230740 at 1730976492669Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1730976492669Flushing stores of hbase:meta,,1.1588230740 at 1730976492669Flushing 1588230740/info: creating writer at 1730976492670 (+1 ms)Flushing 1588230740/info: appending metadata at 1730976492694 (+24 ms)Flushing 1588230740/info: closing flushed file at 1730976492694Flushing 1588230740/ns: creating writer at 1730976492697 (+3 ms)Flushing 1588230740/ns: appending metadata at 1730976492716 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1730976492716Flushing 1588230740/table: creating writer at 1730976492719 (+3 ms)Flushing 1588230740/table: appending metadata at 1730976492738 (+19 ms)Flushing 1588230740/table: closing flushed file at 1730976492738Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@82bb7e5: reopening flushed file at 1730976492741 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3edb9139: reopening flushed file at 1730976492744 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cf8ff79: reopening flushed file at 1730976492746 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=13, compaction requested=false at 1730976492749 (+3 ms)Writing region close event to WAL at 1730976492752 (+3 ms)Running coprocessor post-close hooks at 1730976492756 (+4 ms)Closed at 1730976492756 2024-11-07T10:48:12,756 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T10:48:12,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:12,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45325-0x10019db27800003, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:12,776 INFO [RS:2;10641b2c0a69:45325 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:48:12,776 INFO [RS:2;10641b2c0a69:45325 {}] regionserver.HRegionServer(1031): Exiting; stopping=10641b2c0a69,45325,1730976492042; zookeeper connection closed. 2024-11-07T10:48:12,776 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7307d402 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7307d402 2024-11-07T10:48:12,868 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(976): stopping server 10641b2c0a69,32933,1730976488787; all regions closed. 2024-11-07T10:48:12,871 DEBUG [RS:1;10641b2c0a69:32933 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs 2024-11-07T10:48:12,871 INFO [RS:1;10641b2c0a69:32933 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C32933%2C1730976488787.meta:.meta(num 1730976489683) 2024-11-07T10:48:12,872 DEBUG [RS:1;10641b2c0a69:32933 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/oldWALs 2024-11-07T10:48:12,872 INFO [RS:1;10641b2c0a69:32933 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C32933%2C1730976488787:(num 1730976489326) 2024-11-07T10:48:12,872 DEBUG [RS:1;10641b2c0a69:32933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:48:12,872 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:48:12,872 INFO [RS:1;10641b2c0a69:32933 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:48:12,872 INFO [RS:1;10641b2c0a69:32933 {}] hbase.ChoreService(370): Chore service for: regionserver/10641b2c0a69:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T10:48:12,873 INFO [RS:1;10641b2c0a69:32933 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:48:12,873 INFO [regionserver/10641b2c0a69:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:48:12,873 INFO [RS:1;10641b2c0a69:32933 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32933 2024-11-07T10:48:12,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/10641b2c0a69,32933,1730976488787 2024-11-07T10:48:12,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:48:12,875 INFO [RS:1;10641b2c0a69:32933 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:48:12,877 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [10641b2c0a69,32933,1730976488787] 2024-11-07T10:48:12,878 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/10641b2c0a69,32933,1730976488787 already deleted, retry=false 2024-11-07T10:48:12,878 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 10641b2c0a69,32933,1730976488787 expired; onlineServers=0 2024-11-07T10:48:12,878 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '10641b2c0a69,42249,1730976488690' ***** 2024-11-07T10:48:12,878 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T10:48:12,879 INFO [M:0;10641b2c0a69:42249 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:48:12,879 INFO [M:0;10641b2c0a69:42249 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:48:12,879 DEBUG [M:0;10641b2c0a69:42249 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T10:48:12,879 DEBUG [M:0;10641b2c0a69:42249 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T10:48:12,879 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T10:48:12,879 DEBUG [master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.small.0-1730976489064 {}] cleaner.HFileCleaner(306): Exit Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.small.0-1730976489064,5,FailOnTimeoutGroup] 2024-11-07T10:48:12,879 DEBUG [master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.large.0-1730976489060 {}] cleaner.HFileCleaner(306): Exit Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.large.0-1730976489060,5,FailOnTimeoutGroup] 2024-11-07T10:48:12,879 INFO [M:0;10641b2c0a69:42249 {}] hbase.ChoreService(370): Chore service for: master/10641b2c0a69:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T10:48:12,879 INFO [M:0;10641b2c0a69:42249 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:48:12,879 DEBUG [M:0;10641b2c0a69:42249 {}] master.HMaster(1795): Stopping service threads 2024-11-07T10:48:12,879 INFO [M:0;10641b2c0a69:42249 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T10:48:12,879 INFO [M:0;10641b2c0a69:42249 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:48:12,880 INFO [M:0;10641b2c0a69:42249 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:48:12,880 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T10:48:12,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T10:48:12,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:48:12,881 DEBUG [M:0;10641b2c0a69:42249 {}] zookeeper.ZKUtil(347): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T10:48:12,881 WARN [M:0;10641b2c0a69:42249 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T10:48:12,881 INFO [M:0;10641b2c0a69:42249 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/.lastflushedseqids 2024-11-07T10:48:12,888 INFO [M:0;10641b2c0a69:42249 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:48:12,888 INFO [M:0;10641b2c0a69:42249 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T10:48:12,888 DEBUG [M:0;10641b2c0a69:42249 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T10:48:12,888 INFO [M:0;10641b2c0a69:42249 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:12,888 DEBUG [M:0;10641b2c0a69:42249 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:12,888 DEBUG [M:0;10641b2c0a69:42249 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T10:48:12,888 DEBUG [M:0;10641b2c0a69:42249 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:12,888 INFO [M:0;10641b2c0a69:42249 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-07T10:48:12,898 DEBUG [M:0;10641b2c0a69:42249 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dfc3c1471d3a4f678c9a3238326389c5 2024-11-07T10:48:12,911 DEBUG [M:0;10641b2c0a69:42249 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dfc3c1471d3a4f678c9a3238326389c5 is 82, key is hbase:meta,,1/info:regioninfo/1730976489716/Put/seqid=0 2024-11-07T10:48:12,912 INFO [M:0;10641b2c0a69:42249 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dfc3c1471d3a4f678c9a3238326389c5 2024-11-07T10:48:12,919 DEBUG [M:0;10641b2c0a69:42249 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f9ffedff81114d929e7d40813a912b65 2024-11-07T10:48:12,932 DEBUG [M:0;10641b2c0a69:42249 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f9ffedff81114d929e7d40813a912b65 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1730976490209/Put/seqid=0 2024-11-07T10:48:12,933 INFO [M:0;10641b2c0a69:42249 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f9ffedff81114d929e7d40813a912b65 2024-11-07T10:48:12,941 DEBUG [M:0;10641b2c0a69:42249 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a792d26275fb4aedbdb5746dcde40e4e 2024-11-07T10:48:12,954 DEBUG [M:0;10641b2c0a69:42249 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a792d26275fb4aedbdb5746dcde40e4e is 69, key is 10641b2c0a69,32933,1730976488787/rs:state/1730976489148/Put/seqid=0 2024-11-07T10:48:12,955 INFO [M:0;10641b2c0a69:42249 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a792d26275fb4aedbdb5746dcde40e4e 2024-11-07T10:48:12,957 INFO [M:0;10641b2c0a69:42249 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a792d26275fb4aedbdb5746dcde40e4e 2024-11-07T10:48:12,957 DEBUG [M:0;10641b2c0a69:42249 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dfc3c1471d3a4f678c9a3238326389c5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dfc3c1471d3a4f678c9a3238326389c5 2024-11-07T10:48:12,959 INFO [M:0;10641b2c0a69:42249 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dfc3c1471d3a4f678c9a3238326389c5, entries=8, sequenceid=83, filesize=5.5 K 2024-11-07T10:48:12,959 DEBUG [M:0;10641b2c0a69:42249 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f9ffedff81114d929e7d40813a912b65 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f9ffedff81114d929e7d40813a912b65 2024-11-07T10:48:12,961 INFO [M:0;10641b2c0a69:42249 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f9ffedff81114d929e7d40813a912b65, entries=9, sequenceid=83, filesize=6.4 K 2024-11-07T10:48:12,961 DEBUG [M:0;10641b2c0a69:42249 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a792d26275fb4aedbdb5746dcde40e4e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a792d26275fb4aedbdb5746dcde40e4e 2024-11-07T10:48:12,963 INFO [M:0;10641b2c0a69:42249 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a792d26275fb4aedbdb5746dcde40e4e 2024-11-07T10:48:12,963 INFO [M:0;10641b2c0a69:42249 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abe26705-1ebc-694f-d44a-623a6e33ccba/1e5c4c37-1b63-5950-0b90-d17b5f8c4ae9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a792d26275fb4aedbdb5746dcde40e4e, entries=3, sequenceid=83, filesize=5.3 K 2024-11-07T10:48:12,964 INFO [M:0;10641b2c0a69:42249 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=83, compaction requested=false 2024-11-07T10:48:12,965 INFO [M:0;10641b2c0a69:42249 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:48:12,966 DEBUG [M:0;10641b2c0a69:42249 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730976492888Disabling compacts and flushes for region at 1730976492888Disabling writes for close at 1730976492888Obtaining lock to block concurrent updates at 1730976492888Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730976492888Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1730976492889 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730976492889Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730976492889Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730976492911 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730976492911Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730976492913 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730976492932 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730976492932Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730976492935 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730976492954 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730976492954Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5afb1a8d: reopening flushed file at 1730976492957 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6133542c: reopening flushed file at 1730976492959 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d14e90e: reopening flushed file at 1730976492961 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=83, compaction requested=false at 1730976492964 (+3 ms)Writing region close event to WAL at 1730976492965 (+1 ms)Closed at 1730976492965 2024-11-07T10:48:12,967 INFO [M:0;10641b2c0a69:42249 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T10:48:12,967 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:48:12,967 INFO [M:0;10641b2c0a69:42249 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42249 2024-11-07T10:48:12,967 INFO [M:0;10641b2c0a69:42249 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:48:12,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:12,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32933-0x10019db27800002, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:12,977 INFO [RS:1;10641b2c0a69:32933 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:48:12,977 INFO [RS:1;10641b2c0a69:32933 {}] regionserver.HRegionServer(1031): Exiting; stopping=10641b2c0a69,32933,1730976488787; zookeeper connection closed. 2024-11-07T10:48:12,977 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27f09f63 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27f09f63 2024-11-07T10:48:12,978 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-07T10:48:13,069 INFO [M:0;10641b2c0a69:42249 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:48:13,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:13,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42249-0x10019db27800000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:48:13,072 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T10:48:13,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-07T10:48:13,074 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:48:13,080 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=887 (was 947), ProcessCount=11 (was 12), AvailableMemoryMB=5017 (was 5108)