2024-11-22 19:45:10,448 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 19:45:10,460 main DEBUG Took 0.010499 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 19:45:10,461 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 19:45:10,461 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 19:45:10,462 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 19:45:10,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,470 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 19:45:10,488 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,490 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,491 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,493 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,494 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,496 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,497 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,498 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,499 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,499 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,500 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,501 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,501 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,502 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,502 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 19:45:10,503 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,503 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 19:45:10,505 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 19:45:10,506 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 19:45:10,508 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 19:45:10,509 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 19:45:10,511 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 19:45:10,511 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 19:45:10,523 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 19:45:10,526 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 19:45:10,528 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 19:45:10,529 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 19:45:10,529 main DEBUG createAppenders(={Console}) 2024-11-22 19:45:10,530 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-22 19:45:10,530 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 19:45:10,530 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-22 19:45:10,531 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 19:45:10,532 main DEBUG OutputStream closed 2024-11-22 19:45:10,532 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 19:45:10,533 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 19:45:10,533 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-22 19:45:10,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 19:45:10,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 19:45:10,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 19:45:10,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 19:45:10,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 19:45:10,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 19:45:10,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 19:45:10,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 19:45:10,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 19:45:10,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 19:45:10,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 19:45:10,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 19:45:10,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 19:45:10,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 19:45:10,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 19:45:10,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 19:45:10,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 19:45:10,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 19:45:10,633 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 19:45:10,634 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-22 19:45:10,634 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 19:45:10,635 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-22T19:45:10,650 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-22 19:45:10,652 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 19:45:10,653 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T19:45:10,880 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4 2024-11-22T19:45:10,906 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=276, ProcessCount=11, AvailableMemoryMB=9117 2024-11-22T19:45:11,029 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T19:45:11,123 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:11,140 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/cluster_5029c3a3-f2ea-7856-4561-07caf9c51e15, deleteOnExit=true 2024-11-22T19:45:11,241 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/cluster_5029c3a3-f2ea-7856-4561-07caf9c51e15/zookeeper_0, clientPort=65249, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/cluster_5029c3a3-f2ea-7856-4561-07caf9c51e15/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/cluster_5029c3a3-f2ea-7856-4561-07caf9c51e15/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T19:45:11,255 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65249 2024-11-22T19:45:11,256 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:11,346 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 with version=8 2024-11-22T19:45:11,347 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/hbase-staging 2024-11-22T19:45:11,452 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T19:45:11,734 INFO [Time-limited test {}] client.ConnectionUtils(128): master/982bb8edae1e:0 server-side Connection retries=45 2024-11-22T19:45:11,745 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:11,747 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:11,754 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T19:45:11,754 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:11,755 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T19:45:11,895 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T19:45:11,985 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T19:45:12,023 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T19:45:12,034 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T19:45:12,038 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T19:45:12,082 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15202 (auto-detected) 2024-11-22T19:45:12,083 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T19:45:12,107 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39163 2024-11-22T19:45:12,130 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39163 connecting to ZooKeeper ensemble=127.0.0.1:65249 2024-11-22T19:45:12,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:391630x0, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T19:45:12,170 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39163-0x101413618b90000 connected 2024-11-22T19:45:12,200 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:12,201 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:12,209 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:12,213 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816, hbase.cluster.distributed=false 2024-11-22T19:45:12,241 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T19:45:12,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39163 2024-11-22T19:45:12,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39163 2024-11-22T19:45:12,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39163 2024-11-22T19:45:12,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39163 2024-11-22T19:45:12,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39163 2024-11-22T19:45:12,344 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/982bb8edae1e:0 server-side Connection retries=45 2024-11-22T19:45:12,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:12,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:12,347 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T19:45:12,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:12,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T19:45:12,351 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T19:45:12,354 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T19:45:12,355 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39485 2024-11-22T19:45:12,358 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39485 connecting to ZooKeeper ensemble=127.0.0.1:65249 2024-11-22T19:45:12,359 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:12,360 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:12,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394850x0, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T19:45:12,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39485-0x101413618b90001 connected 2024-11-22T19:45:12,366 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:12,372 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T19:45:12,384 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.cache 2024-11-22T19:45:12,393 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T19:45:12,394 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T19:45:12,396 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T19:45:12,399 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T19:45:12,399 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T19:45:12,405 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T19:45:12,407 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T19:45:12,407 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.persistence is complete. 2024-11-22T19:45:12,410 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T19:45:12,412 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T19:45:12,418 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T19:45:12,419 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39485 2024-11-22T19:45:12,419 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39485 2024-11-22T19:45:12,421 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39485 2024-11-22T19:45:12,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39485 2024-11-22T19:45:12,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39485 2024-11-22T19:45:12,454 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/982bb8edae1e:0 server-side Connection retries=45 2024-11-22T19:45:12,454 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:12,455 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:12,455 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T19:45:12,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:12,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T19:45:12,456 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T19:45:12,457 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T19:45:12,458 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44433 2024-11-22T19:45:12,460 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44433 connecting to ZooKeeper ensemble=127.0.0.1:65249 2024-11-22T19:45:12,460 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:12,461 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:12,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444330x0, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T19:45:12,467 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:444330x0, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:12,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44433-0x101413618b90002 connected 2024-11-22T19:45:12,468 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T19:45:12,469 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.cache 2024-11-22T19:45:12,470 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T19:45:12,470 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T19:45:12,470 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T19:45:12,472 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T19:45:12,473 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T19:45:12,473 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T19:45:12,473 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T19:45:12,474 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.persistence is complete. 2024-11-22T19:45:12,476 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T19:45:12,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T19:45:12,481 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T19:45:12,489 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44433 2024-11-22T19:45:12,489 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44433 2024-11-22T19:45:12,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44433 2024-11-22T19:45:12,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44433 2024-11-22T19:45:12,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44433 2024-11-22T19:45:12,511 DEBUG [M:0;982bb8edae1e:39163 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;982bb8edae1e:39163 2024-11-22T19:45:12,512 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/982bb8edae1e,39163,1732304711532 2024-11-22T19:45:12,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:12,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:12,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:12,523 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/982bb8edae1e,39163,1732304711532 2024-11-22T19:45:12,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T19:45:12,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T19:45:12,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:12,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:12,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:12,550 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T19:45:12,552 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/982bb8edae1e,39163,1732304711532 from backup master directory 2024-11-22T19:45:12,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:12,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:12,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/982bb8edae1e,39163,1732304711532 2024-11-22T19:45:12,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:12,557 WARN [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T19:45:12,557 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=982bb8edae1e,39163,1732304711532 2024-11-22T19:45:12,560 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T19:45:12,562 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T19:45:12,572 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/hbase.id] with ID: 7e086ae3-97f4-4e89-b1aa-264b8458342f 2024-11-22T19:45:12,573 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/.tmp/hbase.id 2024-11-22T19:45:12,581 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/hbase.id] 2024-11-22T19:45:12,625 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:12,631 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T19:45:12,651 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-22T19:45:12,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:12,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:12,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:12,684 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T19:45:12,687 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T19:45:12,734 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T19:45:12,740 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T19:45:12,798 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store 2024-11-22T19:45:12,825 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T19:45:12,829 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:12,831 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T19:45:12,831 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:12,832 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:12,834 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T19:45:12,834 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:12,835 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:12,836 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732304712831Disabling compacts and flushes for region at 1732304712831Disabling writes for close at 1732304712834 (+3 ms)Writing region close event to WAL at 1732304712835 (+1 ms)Closed at 1732304712835 2024-11-22T19:45:12,836 WARN [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/.initializing 2024-11-22T19:45:12,837 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/WALs/982bb8edae1e,39163,1732304711532 2024-11-22T19:45:12,851 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T19:45:12,866 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=982bb8edae1e%2C39163%2C1732304711532, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/WALs/982bb8edae1e,39163,1732304711532, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/oldWALs, maxLogs=10 2024-11-22T19:45:12,909 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/WALs/982bb8edae1e,39163,1732304711532/982bb8edae1e%2C39163%2C1732304711532.1732304712872 2024-11-22T19:45:12,910 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T19:45:12,910 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T19:45:12,911 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:12,915 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:12,916 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:12,956 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:12,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T19:45:12,998 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:13,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:13,001 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:13,007 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T19:45:13,007 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:13,008 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:13,008 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:13,013 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T19:45:13,014 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:13,014 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:13,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:13,020 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T19:45:13,020 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:13,021 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:13,021 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:13,025 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:13,026 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:13,029 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:13,030 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:13,032 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T19:45:13,034 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:13,043 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T19:45:13,043 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74939439, jitterRate=0.11668466031551361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T19:45:13,050 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732304712930Initializing all the Stores at 1732304712930Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304712931 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304712932 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304712932Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304712932Cleaning up temporary data from old regions at 1732304713030 (+98 ms)Region opened successfully at 1732304713050 (+20 ms) 2024-11-22T19:45:13,052 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T19:45:13,090 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c34e25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=982bb8edae1e/172.17.0.2:0 2024-11-22T19:45:13,125 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T19:45:13,138 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T19:45:13,138 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T19:45:13,141 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T19:45:13,142 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T19:45:13,148 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T19:45:13,148 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T19:45:13,175 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T19:45:13,184 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T19:45:13,187 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T19:45:13,189 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T19:45:13,191 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T19:45:13,193 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T19:45:13,196 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T19:45:13,199 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T19:45:13,201 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T19:45:13,203 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T19:45:13,205 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T19:45:13,224 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T19:45:13,226 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T19:45:13,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:13,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:13,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:13,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,244 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=982bb8edae1e,39163,1732304711532, sessionid=0x101413618b90000, setting cluster-up flag (Was=false) 2024-11-22T19:45:13,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,267 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T19:45:13,269 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=982bb8edae1e,39163,1732304711532 2024-11-22T19:45:13,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,280 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T19:45:13,281 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=982bb8edae1e,39163,1732304711532 2024-11-22T19:45:13,287 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T19:45:13,299 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(746): ClusterId : 7e086ae3-97f4-4e89-b1aa-264b8458342f 2024-11-22T19:45:13,299 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(746): ClusterId : 7e086ae3-97f4-4e89-b1aa-264b8458342f 2024-11-22T19:45:13,302 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T19:45:13,303 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T19:45:13,308 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T19:45:13,308 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T19:45:13,309 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T19:45:13,309 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T19:45:13,314 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T19:45:13,314 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T19:45:13,315 DEBUG [RS:1;982bb8edae1e:44433 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57427ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=982bb8edae1e/172.17.0.2:0 2024-11-22T19:45:13,315 DEBUG [RS:0;982bb8edae1e:39485 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e54717e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=982bb8edae1e/172.17.0.2:0 2024-11-22T19:45:13,333 DEBUG [RS:1;982bb8edae1e:44433 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;982bb8edae1e:44433 2024-11-22T19:45:13,334 DEBUG [RS:0;982bb8edae1e:39485 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;982bb8edae1e:39485 2024-11-22T19:45:13,338 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T19:45:13,338 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T19:45:13,338 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T19:45:13,338 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T19:45:13,338 DEBUG [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T19:45:13,338 DEBUG [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T19:45:13,342 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(2659): reportForDuty to master=982bb8edae1e,39163,1732304711532 with port=39485, startcode=1732304712304 2024-11-22T19:45:13,342 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(2659): reportForDuty to master=982bb8edae1e,39163,1732304711532 with port=44433, startcode=1732304712453 2024-11-22T19:45:13,360 DEBUG [RS:1;982bb8edae1e:44433 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T19:45:13,360 DEBUG [RS:0;982bb8edae1e:39485 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T19:45:13,383 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T19:45:13,395 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T19:45:13,403 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T19:45:13,410 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 982bb8edae1e,39163,1732304711532 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T19:45:13,420 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/982bb8edae1e:0, corePoolSize=5, maxPoolSize=5 2024-11-22T19:45:13,421 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/982bb8edae1e:0, corePoolSize=5, maxPoolSize=5 2024-11-22T19:45:13,421 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/982bb8edae1e:0, corePoolSize=5, maxPoolSize=5 2024-11-22T19:45:13,421 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/982bb8edae1e:0, corePoolSize=5, maxPoolSize=5 2024-11-22T19:45:13,421 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/982bb8edae1e:0, corePoolSize=10, maxPoolSize=10 2024-11-22T19:45:13,422 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,422 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/982bb8edae1e:0, corePoolSize=2, maxPoolSize=2 2024-11-22T19:45:13,422 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,427 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732304743427 2024-11-22T19:45:13,430 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T19:45:13,430 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T19:45:13,430 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T19:45:13,431 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T19:45:13,435 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T19:45:13,436 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T19:45:13,436 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T19:45:13,436 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T19:45:13,437 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:13,437 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T19:45:13,438 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,443 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T19:45:13,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35217, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T19:45:13,443 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44697, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T19:45:13,444 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T19:45:13,445 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T19:45:13,448 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T19:45:13,448 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T19:45:13,450 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.large.0-1732304713450,5,FailOnTimeoutGroup] 2024-11-22T19:45:13,451 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.small.0-1732304713451,5,FailOnTimeoutGroup] 2024-11-22T19:45:13,451 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,452 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T19:45:13,452 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39163 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T19:45:13,453 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,454 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,459 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39163 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 982bb8edae1e,44433,1732304712453 2024-11-22T19:45:13,462 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39163 {}] master.ServerManager(517): Registering regionserver=982bb8edae1e,44433,1732304712453 2024-11-22T19:45:13,466 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T19:45:13,466 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 2024-11-22T19:45:13,472 DEBUG [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 2024-11-22T19:45:13,473 DEBUG [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T19:45:13,473 DEBUG [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T19:45:13,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:13,480 DEBUG [RS:1;982bb8edae1e:44433 {}] zookeeper.ZKUtil(111): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/982bb8edae1e,44433,1732304712453 2024-11-22T19:45:13,480 WARN [RS:1;982bb8edae1e:44433 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T19:45:13,480 INFO [RS:1;982bb8edae1e:44433 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T19:45:13,481 DEBUG [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,44433,1732304712453 2024-11-22T19:45:13,486 DEBUG [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-22T19:45:13,486 WARN [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-22T19:45:13,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:13,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T19:45:13,492 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [982bb8edae1e,44433,1732304712453] 2024-11-22T19:45:13,498 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T19:45:13,498 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:13,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:13,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T19:45:13,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T19:45:13,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:13,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:13,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T19:45:13,507 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T19:45:13,507 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:13,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:13,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T19:45:13,511 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T19:45:13,511 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:13,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:13,512 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T19:45:13,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740 2024-11-22T19:45:13,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740 2024-11-22T19:45:13,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T19:45:13,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T19:45:13,514 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T19:45:13,514 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T19:45:13,521 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T19:45:13,524 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T19:45:13,524 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69557418, jitterRate=0.036486297845840454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T19:45:13,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732304713491Initializing all the Stores at 1732304713491Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304713491Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304713491Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304713491Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304713491Cleaning up temporary data from old regions at 1732304713514 (+23 ms)Region opened successfully at 1732304713528 (+14 ms) 2024-11-22T19:45:13,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T19:45:13,529 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T19:45:13,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T19:45:13,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T19:45:13,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T19:45:13,531 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T19:45:13,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732304713529Disabling compacts and flushes for region at 1732304713529Disabling writes for close at 1732304713529Writing region close event to WAL at 1732304713530 (+1 ms)Closed at 1732304713531 (+1 ms) 2024-11-22T19:45:13,534 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T19:45:13,534 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T19:45:13,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T19:45:13,542 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T19:45:13,549 INFO [RS:1;982bb8edae1e:44433 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T19:45:13,549 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,550 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T19:45:13,553 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T19:45:13,555 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T19:45:13,556 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T19:45:13,558 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,558 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,558 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,559 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,559 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,559 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,559 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/982bb8edae1e:0, corePoolSize=2, maxPoolSize=2 2024-11-22T19:45:13,559 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,560 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,560 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,560 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,560 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,561 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,561 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:13,561 DEBUG [RS:1;982bb8edae1e:44433 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:13,562 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,562 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,563 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,563 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,563 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,563 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,44433,1732304712453-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T19:45:13,585 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T19:45:13,587 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,44433,1732304712453-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,587 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,587 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.Replication(171): 982bb8edae1e,44433,1732304712453 started 2024-11-22T19:45:13,588 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(2659): reportForDuty to master=982bb8edae1e,39163,1732304711532 with port=39485, startcode=1732304712304 2024-11-22T19:45:13,590 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39163 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 982bb8edae1e,39485,1732304712304 2024-11-22T19:45:13,590 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39163 {}] master.ServerManager(517): Registering regionserver=982bb8edae1e,39485,1732304712304 2024-11-22T19:45:13,593 DEBUG [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 2024-11-22T19:45:13,593 DEBUG [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T19:45:13,593 DEBUG [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T19:45:13,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:13,600 DEBUG [RS:0;982bb8edae1e:39485 {}] zookeeper.ZKUtil(111): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/982bb8edae1e,39485,1732304712304 2024-11-22T19:45:13,600 WARN [RS:0;982bb8edae1e:39485 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T19:45:13,600 INFO [RS:0;982bb8edae1e:39485 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T19:45:13,600 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [982bb8edae1e,39485,1732304712304] 2024-11-22T19:45:13,601 DEBUG [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,39485,1732304712304 2024-11-22T19:45:13,607 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T19:45:13,612 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T19:45:13,613 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,613 INFO [RS:0;982bb8edae1e:39485 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T19:45:13,613 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(1482): Serving as 982bb8edae1e,44433,1732304712453, RpcServer on 982bb8edae1e/172.17.0.2:44433, sessionid=0x101413618b90002 2024-11-22T19:45:13,613 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,614 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T19:45:13,614 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T19:45:13,615 DEBUG [RS:1;982bb8edae1e:44433 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 982bb8edae1e,44433,1732304712453 2024-11-22T19:45:13,615 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T19:45:13,615 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,44433,1732304712453' 2024-11-22T19:45:13,615 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,615 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T19:45:13,615 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,616 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,616 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,616 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,616 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,616 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T19:45:13,616 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/982bb8edae1e:0, corePoolSize=2, maxPoolSize=2 2024-11-22T19:45:13,617 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,617 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,617 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,617 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T19:45:13,617 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T19:45:13,617 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,617 DEBUG [RS:1;982bb8edae1e:44433 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 982bb8edae1e,44433,1732304712453 2024-11-22T19:45:13,617 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,618 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,44433,1732304712453' 2024-11-22T19:45:13,618 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T19:45:13,618 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:13,618 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:13,618 DEBUG [RS:0;982bb8edae1e:39485 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:13,618 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T19:45:13,619 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,619 DEBUG [RS:1;982bb8edae1e:44433 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T19:45:13,619 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,619 INFO [RS:1;982bb8edae1e:44433 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T19:45:13,619 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,619 INFO [RS:1;982bb8edae1e:44433 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T19:45:13,619 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,620 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,620 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,39485,1732304712304-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T19:45:13,638 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T19:45:13,638 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,39485,1732304712304-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,638 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,638 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.Replication(171): 982bb8edae1e,39485,1732304712304 started 2024-11-22T19:45:13,655 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:13,656 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(1482): Serving as 982bb8edae1e,39485,1732304712304, RpcServer on 982bb8edae1e/172.17.0.2:39485, sessionid=0x101413618b90001 2024-11-22T19:45:13,656 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T19:45:13,656 DEBUG [RS:0;982bb8edae1e:39485 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 982bb8edae1e,39485,1732304712304 2024-11-22T19:45:13,656 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,39485,1732304712304' 2024-11-22T19:45:13,656 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T19:45:13,657 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T19:45:13,658 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T19:45:13,658 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T19:45:13,658 DEBUG [RS:0;982bb8edae1e:39485 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 982bb8edae1e,39485,1732304712304 2024-11-22T19:45:13,658 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,39485,1732304712304' 2024-11-22T19:45:13,658 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T19:45:13,659 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T19:45:13,659 DEBUG [RS:0;982bb8edae1e:39485 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T19:45:13,659 INFO [RS:0;982bb8edae1e:39485 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T19:45:13,660 INFO [RS:0;982bb8edae1e:39485 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T19:45:13,707 WARN [982bb8edae1e:39163 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T19:45:13,727 INFO [RS:1;982bb8edae1e:44433 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T19:45:13,729 INFO [RS:1;982bb8edae1e:44433 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=982bb8edae1e%2C44433%2C1732304712453, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,44433,1732304712453, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs, maxLogs=55 2024-11-22T19:45:13,751 INFO [RS:1;982bb8edae1e:44433 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,44433,1732304712453/982bb8edae1e%2C44433%2C1732304712453.1732304713731 2024-11-22T19:45:13,751 DEBUG [RS:1;982bb8edae1e:44433 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T19:45:13,761 INFO [RS:0;982bb8edae1e:39485 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T19:45:13,762 INFO [RS:0;982bb8edae1e:39485 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=982bb8edae1e%2C39485%2C1732304712304, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,39485,1732304712304, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs, maxLogs=55 2024-11-22T19:45:13,782 INFO [RS:0;982bb8edae1e:39485 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,39485,1732304712304/982bb8edae1e%2C39485%2C1732304712304.1732304713763 2024-11-22T19:45:13,782 DEBUG [RS:0;982bb8edae1e:39485 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T19:45:13,960 DEBUG [982bb8edae1e:39163 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T19:45:13,971 DEBUG [982bb8edae1e:39163 {}] balancer.BalancerClusterState(204): Hosts are {982bb8edae1e=0} racks are {/default-rack=0} 2024-11-22T19:45:13,975 DEBUG [982bb8edae1e:39163 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T19:45:13,975 DEBUG [982bb8edae1e:39163 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T19:45:13,975 DEBUG [982bb8edae1e:39163 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T19:45:13,975 DEBUG [982bb8edae1e:39163 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T19:45:13,975 INFO [982bb8edae1e:39163 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T19:45:13,975 INFO [982bb8edae1e:39163 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T19:45:13,975 DEBUG [982bb8edae1e:39163 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T19:45:13,983 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=982bb8edae1e,44433,1732304712453 2024-11-22T19:45:13,988 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 982bb8edae1e,44433,1732304712453, state=OPENING 2024-11-22T19:45:13,995 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T19:45:13,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:13,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:13,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:13,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:13,999 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T19:45:14,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=982bb8edae1e,44433,1732304712453}] 2024-11-22T19:45:14,173 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T19:45:14,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37885, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T19:45:14,188 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T19:45:14,188 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T19:45:14,189 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T19:45:14,190 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=982bb8edae1e%2C44433%2C1732304712453.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,44433,1732304712453, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs, maxLogs=55 2024-11-22T19:45:14,211 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,44433,1732304712453/982bb8edae1e%2C44433%2C1732304712453.meta.1732304714192.meta 2024-11-22T19:45:14,211 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T19:45:14,212 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T19:45:14,213 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T19:45:14,216 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T19:45:14,220 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T19:45:14,225 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T19:45:14,225 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:14,225 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T19:45:14,226 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T19:45:14,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T19:45:14,228 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T19:45:14,228 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:14,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:14,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T19:45:14,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T19:45:14,230 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:14,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:14,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T19:45:14,231 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T19:45:14,231 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:14,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:14,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T19:45:14,232 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T19:45:14,232 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:14,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:14,232 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T19:45:14,233 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740 2024-11-22T19:45:14,234 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740 2024-11-22T19:45:14,234 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T19:45:14,235 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T19:45:14,235 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T19:45:14,236 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T19:45:14,237 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65969181, jitterRate=-0.01698260009288788}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T19:45:14,237 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T19:45:14,238 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732304714226Writing region info on filesystem at 1732304714226Initializing all the Stores at 1732304714226Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304714227 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304714227Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304714227Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304714227Cleaning up temporary data from old regions at 1732304714235 (+8 ms)Running coprocessor post-open hooks at 1732304714237 (+2 ms)Region opened successfully at 1732304714238 (+1 ms) 2024-11-22T19:45:14,245 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732304714165 2024-11-22T19:45:14,255 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T19:45:14,255 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T19:45:14,256 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=982bb8edae1e,44433,1732304712453 2024-11-22T19:45:14,257 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 982bb8edae1e,44433,1732304712453, state=OPEN 2024-11-22T19:45:14,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T19:45:14,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T19:45:14,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T19:45:14,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:14,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:14,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:14,260 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=982bb8edae1e,44433,1732304712453 2024-11-22T19:45:14,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T19:45:14,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=982bb8edae1e,44433,1732304712453 in 261 msec 2024-11-22T19:45:14,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T19:45:14,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 723 msec 2024-11-22T19:45:14,268 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T19:45:14,268 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T19:45:14,289 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T19:45:14,290 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=982bb8edae1e,44433,1732304712453, seqNum=-1] 2024-11-22T19:45:14,310 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T19:45:14,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T19:45:14,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0010 sec 2024-11-22T19:45:14,329 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732304714328, completionTime=-1 2024-11-22T19:45:14,331 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T19:45:14,331 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T19:45:14,358 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T19:45:14,358 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732304774358 2024-11-22T19:45:14,358 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732304834358 2024-11-22T19:45:14,358 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-22T19:45:14,360 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-22T19:45:14,366 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,39163,1732304711532-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:14,367 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,39163,1732304711532-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:14,367 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,39163,1732304711532-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:14,368 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-982bb8edae1e:39163, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:14,369 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:14,369 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:14,375 DEBUG [master/982bb8edae1e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T19:45:14,396 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.838sec 2024-11-22T19:45:14,398 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T19:45:14,399 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T19:45:14,400 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T19:45:14,401 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T19:45:14,401 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T19:45:14,402 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,39163,1732304711532-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T19:45:14,402 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,39163,1732304711532-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T19:45:14,403 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T19:45:14,404 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T19:45:14,405 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,39163,1732304711532-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:14,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32be4008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T19:45:14,428 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T19:45:14,428 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T19:45:14,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 982bb8edae1e,39163,-1 for getting cluster id 2024-11-22T19:45:14,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T19:45:14,445 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e086ae3-97f4-4e89-b1aa-264b8458342f' 2024-11-22T19:45:14,447 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T19:45:14,448 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e086ae3-97f4-4e89-b1aa-264b8458342f" 2024-11-22T19:45:14,448 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7042e567, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T19:45:14,449 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [982bb8edae1e,39163,-1] 2024-11-22T19:45:14,451 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T19:45:14,453 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:14,454 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T19:45:14,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1061b362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T19:45:14,458 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T19:45:14,465 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=982bb8edae1e,44433,1732304712453, seqNum=-1] 2024-11-22T19:45:14,465 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T19:45:14,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T19:45:14,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=982bb8edae1e,39163,1732304711532 2024-11-22T19:45:14,513 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T19:45:14,519 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 982bb8edae1e,39163,1732304711532 2024-11-22T19:45:14,522 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@635544c 2024-11-22T19:45:14,523 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T19:45:14,525 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T19:45:14,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732304714507', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T19:45:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732304714507 2024-11-22T19:45:14,538 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732304714507 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T19:45:14,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732304714507" procId is: 4 2024-11-22T19:45:14,541 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:14,543 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732304714507 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T19:45:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T19:45:14,557 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732304714507-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ffec03947c9b817791870e0129146756, NAME => 'testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732304714507', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 2024-11-22T19:45:14,563 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732304714507-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:14,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732304714507-pool-0 {}] regionserver.HRegion(1722): Closing ffec03947c9b817791870e0129146756, disabling compactions & flushes 2024-11-22T19:45:14,564 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732304714507-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:14,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732304714507-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:14,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732304714507-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. after waiting 0 ms 2024-11-22T19:45:14,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732304714507-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:14,564 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732304714507-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:14,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732304714507-pool-0 {}] regionserver.HRegion(1676): Region close journal for ffec03947c9b817791870e0129146756: Waiting for close lock at 1732304714564Disabling compacts and flushes for region at 1732304714564Disabling writes for close at 1732304714564Writing region close event to WAL at 1732304714564Closed at 1732304714564 2024-11-22T19:45:14,565 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732304714507 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T19:45:14,570 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732304714565"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732304714565"}]},"ts":"1732304714565"} 2024-11-22T19:45:14,574 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T19:45:14,575 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732304714507 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T19:45:14,578 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732304714507","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732304714575"}]},"ts":"1732304714575"} 2024-11-22T19:45:14,582 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732304714507, state=ENABLING in hbase:meta 2024-11-22T19:45:14,582 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(204): Hosts are {982bb8edae1e=0} racks are {/default-rack=0} 2024-11-22T19:45:14,583 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T19:45:14,583 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T19:45:14,583 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T19:45:14,583 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T19:45:14,583 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T19:45:14,583 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T19:45:14,583 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T19:45:14,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, ASSIGN}] 2024-11-22T19:45:14,585 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, ASSIGN 2024-11-22T19:45:14,586 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, ASSIGN; state=OFFLINE, location=982bb8edae1e,39485,1732304712304; forceNewPlan=false, retain=false 2024-11-22T19:45:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T19:45:14,739 INFO [982bb8edae1e:39163 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T19:45:14,740 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ffec03947c9b817791870e0129146756, regionState=OPENING, regionLocation=982bb8edae1e,39485,1732304712304 2024-11-22T19:45:14,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, ASSIGN because future has completed 2024-11-22T19:45:14,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffec03947c9b817791870e0129146756, server=982bb8edae1e,39485,1732304712304}] 2024-11-22T19:45:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T19:45:14,897 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T19:45:14,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51625, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T19:45:14,901 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:14,902 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ffec03947c9b817791870e0129146756, NAME => 'testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.', STARTKEY => '', ENDKEY => ''} 2024-11-22T19:45:14,902 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732304714507 ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,902 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:14,902 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,902 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,903 INFO [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,907 INFO [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffec03947c9b817791870e0129146756 columnFamilyName family 2024-11-22T19:45:14,907 DEBUG [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:14,908 INFO [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] regionserver.HStore(327): Store=ffec03947c9b817791870e0129146756/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:14,908 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,909 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,909 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,910 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,910 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,911 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,919 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T19:45:14,919 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ffec03947c9b817791870e0129146756; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59889776, jitterRate=-0.10757279396057129}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T19:45:14,919 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffec03947c9b817791870e0129146756 2024-11-22T19:45:14,920 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ffec03947c9b817791870e0129146756: Running coprocessor pre-open hook at 1732304714903Writing region info on filesystem at 1732304714903Initializing all the Stores at 1732304714903Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304714903Cleaning up temporary data from old regions at 1732304714910 (+7 ms)Running coprocessor post-open hooks at 1732304714919 (+9 ms)Region opened successfully at 1732304714920 (+1 ms) 2024-11-22T19:45:14,921 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756., pid=6, masterSystemTime=1732304714896 2024-11-22T19:45:14,924 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:14,924 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:14,925 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ffec03947c9b817791870e0129146756, regionState=OPEN, openSeqNum=2, regionLocation=982bb8edae1e,39485,1732304712304 2024-11-22T19:45:14,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffec03947c9b817791870e0129146756, server=982bb8edae1e,39485,1732304712304 because future has completed 2024-11-22T19:45:14,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T19:45:14,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ffec03947c9b817791870e0129146756, server=982bb8edae1e,39485,1732304712304 in 188 msec 2024-11-22T19:45:14,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T19:45:14,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, ASSIGN in 351 msec 2024-11-22T19:45:14,938 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732304714507 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T19:45:14,938 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732304714507","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732304714938"}]},"ts":"1732304714938"} 2024-11-22T19:45:14,941 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732304714507, state=ENABLED in hbase:meta 2024-11-22T19:45:14,942 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732304714507 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T19:45:14,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732304714507 in 409 msec 2024-11-22T19:45:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T19:45:15,173 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732304714507 get assigned. Timeout = 60000ms 2024-11-22T19:45:15,173 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732304714507 completed 2024-11-22T19:45:15,176 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T19:45:15,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732304714507 assigned to meta. Checking AM states. 2024-11-22T19:45:15,183 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T19:45:15,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732304714507 assigned. 2024-11-22T19:45:15,191 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732304714507', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756., hostname=982bb8edae1e,39485,1732304712304, seqNum=2] 2024-11-22T19:45:15,192 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T19:45:15,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T19:45:15,204 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ffec03947c9b817791870e0129146756 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T19:45:15,231 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a072ec1126d24efe82bdf1744dea8ea3 2024-11-22T19:45:15,274 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/.tmp/family/a072ec1126d24efe82bdf1744dea8ea3 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T19:45:15,279 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/.tmp/family/a072ec1126d24efe82bdf1744dea8ea3 2024-11-22T19:45:15,316 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/.tmp/family/a072ec1126d24efe82bdf1744dea8ea3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3 2024-11-22T19:45:15,318 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3, delay=910 ms 2024-11-22T19:45:15,320 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T19:45:15,325 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ffec03947c9b817791870e0129146756 in 122ms, sequenceid=6, compaction requested=false 2024-11-22T19:45:15,326 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732304714507' 2024-11-22T19:45:15,329 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ffec03947c9b817791870e0129146756: 2024-11-22T19:45:16,230 DEBUG [hfile-prefetch-1732304715318 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a072ec1126d24efe82bdf1744dea8ea3_0. 2024-11-22T19:45:16,234 DEBUG [hfile-prefetch-1732304715318 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a072ec1126d24efe82bdf1744dea8ea3_117. 2024-11-22T19:45:16,235 DEBUG [hfile-prefetch-1732304715318 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3, with total blocks 2, and data blocks 1 2024-11-22T19:45:16,235 DEBUG [hfile-prefetch-1732304715318 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T19:45:16,236 DEBUG [hfile-prefetch-1732304715318 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T19:45:16,236 DEBUG [hfile-prefetch-1732304715318 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3. Will sleep 100ms and try the verification again. 2024-11-22T19:45:16,275 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T19:45:16,275 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T19:45:16,275 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T19:45:16,275 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T19:45:16,290 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T19:45:16,290 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732304716275 2024-11-22T19:45:16,290 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T19:45:16,336 DEBUG [hfile-prefetch-1732304715318 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3, with total blocks 2, and data blocks 1 2024-11-22T19:45:16,336 DEBUG [hfile-prefetch-1732304715318 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-22T19:45:16,336 DEBUG [hfile-prefetch-1732304715318 {}] bucket.BucketCache(2293): found block for file a072ec1126d24efe82bdf1744dea8ea3 in the backing map. Acquiring read lock for offset 0 2024-11-22T19:45:16,337 DEBUG [hfile-prefetch-1732304715318 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3 has now been fully cached. 2024-11-22T19:45:16,337 DEBUG [hfile-prefetch-1732304715318 {}] hfile.PrefetchExecutor(121): Prefetch completed for a072ec1126d24efe82bdf1744dea8ea3 2024-11-22T19:45:16,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=ffec03947c9b817791870e0129146756, source=982bb8edae1e,39485,1732304712304, destination=982bb8edae1e,44433,1732304712453, warming up region on 982bb8edae1e,44433,1732304712453 2024-11-22T19:45:16,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=ffec03947c9b817791870e0129146756, source=982bb8edae1e,39485,1732304712304, destination=982bb8edae1e,44433,1732304712453, running balancer 2024-11-22T19:45:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, REOPEN/MOVE 2024-11-22T19:45:16,343 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, REOPEN/MOVE 2024-11-22T19:45:16,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.HRegion(7855): Warmup {ENCODED => ffec03947c9b817791870e0129146756, NAME => 'testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.', STARTKEY => '', ENDKEY => ''} 2024-11-22T19:45:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:16,344 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ffec03947c9b817791870e0129146756, regionState=CLOSING, regionLocation=982bb8edae1e,39485,1732304712304 2024-11-22T19:45:16,344 INFO [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,345 INFO [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffec03947c9b817791870e0129146756 columnFamilyName family 2024-11-22T19:45:16,345 DEBUG [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:16,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, REOPEN/MOVE because future has completed 2024-11-22T19:45:16,348 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-22T19:45:16,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffec03947c9b817791870e0129146756, server=982bb8edae1e,39485,1732304712304}] 2024-11-22T19:45:16,348 DEBUG [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3 2024-11-22T19:45:16,349 INFO [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] regionserver.HStore(327): Store=ffec03947c9b817791870e0129146756/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.HRegion(1722): Closing ffec03947c9b817791870e0129146756, disabling compactions & flushes 2024-11-22T19:45:16,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. after waiting 0 ms 2024-11-22T19:45:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] regionserver.HRegion(1676): Region close journal for ffec03947c9b817791870e0129146756: Waiting for close lock at 1732304716349Disabling compacts and flushes for region at 1732304716349Disabling writes for close at 1732304716349Writing region close event to WAL at 1732304716350 (+1 ms)Closed at 1732304716350 2024-11-22T19:45:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-22T19:45:16,505 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,506 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-22T19:45:16,506 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing ffec03947c9b817791870e0129146756, disabling compactions & flushes 2024-11-22T19:45:16,506 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,506 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,506 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. after waiting 0 ms 2024-11-22T19:45:16,507 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,508 DEBUG [StoreFileCloser-ffec03947c9b817791870e0129146756-family-1 {}] bucket.BucketCache(768): Removing file a072ec1126d24efe82bdf1744dea8ea3 for region ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,508 DEBUG [StoreFileCloser-ffec03947c9b817791870e0129146756-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file a072ec1126d24efe82bdf1744dea8ea3, starting offset: 0, end offset: 9223372036854775807 2024-11-22T19:45:16,510 DEBUG [StoreFileCloser-ffec03947c9b817791870e0129146756-family-1 {}] bucket.BucketCache(856): removed key a072ec1126d24efe82bdf1744dea8ea3_0 from back map with offset lock 2094080 in the evict process 2024-11-22T19:45:16,515 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T19:45:16,516 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,516 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for ffec03947c9b817791870e0129146756: Waiting for close lock at 1732304716506Running coprocessor pre-close hooks at 1732304716506Disabling compacts and flushes for region at 1732304716506Disabling writes for close at 1732304716506Writing region close event to WAL at 1732304716510 (+4 ms)Running coprocessor post-close hooks at 1732304716515 (+5 ms)Closed at 1732304716516 (+1 ms) 2024-11-22T19:45:16,517 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding ffec03947c9b817791870e0129146756 move to 982bb8edae1e,44433,1732304712453 record at close sequenceid=6 2024-11-22T19:45:16,520 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,520 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ffec03947c9b817791870e0129146756, regionState=CLOSED 2024-11-22T19:45:16,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffec03947c9b817791870e0129146756, server=982bb8edae1e,39485,1732304712304 because future has completed 2024-11-22T19:45:16,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T19:45:16,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure ffec03947c9b817791870e0129146756, server=982bb8edae1e,39485,1732304712304 in 176 msec 2024-11-22T19:45:16,527 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, REOPEN/MOVE; state=CLOSED, location=982bb8edae1e,44433,1732304712453; forceNewPlan=false, retain=false 2024-11-22T19:45:16,591 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T19:45:16,591 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T19:45:16,591 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T19:45:16,591 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T19:45:16,594 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T19:45:16,594 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732304716591 2024-11-22T19:45:16,594 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T19:45:16,678 INFO [982bb8edae1e:39163 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T19:45:16,679 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ffec03947c9b817791870e0129146756, regionState=OPENING, regionLocation=982bb8edae1e,44433,1732304712453 2024-11-22T19:45:16,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, REOPEN/MOVE because future has completed 2024-11-22T19:45:16,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffec03947c9b817791870e0129146756, server=982bb8edae1e,44433,1732304712453}] 2024-11-22T19:45:16,837 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,838 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ffec03947c9b817791870e0129146756, NAME => 'testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.', STARTKEY => '', ENDKEY => ''} 2024-11-22T19:45:16,838 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732304714507 ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,838 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:16,839 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,839 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,840 INFO [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,841 INFO [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffec03947c9b817791870e0129146756 columnFamilyName family 2024-11-22T19:45:16,841 DEBUG [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:16,843 DEBUG [StoreFileOpener-ffec03947c9b817791870e0129146756-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3, delay=988 ms 2024-11-22T19:45:16,844 DEBUG [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3 2024-11-22T19:45:16,845 INFO [StoreOpener-ffec03947c9b817791870e0129146756-1 {}] regionserver.HStore(327): Store=ffec03947c9b817791870e0129146756/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:16,845 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,846 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,846 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,846 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,846 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,851 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,851 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ffec03947c9b817791870e0129146756; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70831853, jitterRate=0.05547685921192169}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T19:45:16,851 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,852 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ffec03947c9b817791870e0129146756: Running coprocessor pre-open hook at 1732304716839Writing region info on filesystem at 1732304716839Initializing all the Stores at 1732304716839Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304716839Cleaning up temporary data from old regions at 1732304716846 (+7 ms)Running coprocessor post-open hooks at 1732304716851 (+5 ms)Region opened successfully at 1732304716852 (+1 ms) 2024-11-22T19:45:16,853 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756., pid=9, masterSystemTime=1732304716834 2024-11-22T19:45:16,856 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,856 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ffec03947c9b817791870e0129146756, regionState=OPEN, openSeqNum=10, regionLocation=982bb8edae1e,44433,1732304712453 2024-11-22T19:45:16,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffec03947c9b817791870e0129146756, server=982bb8edae1e,44433,1732304712453 because future has completed 2024-11-22T19:45:16,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-22T19:45:16,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ffec03947c9b817791870e0129146756, server=982bb8edae1e,44433,1732304712453 in 179 msec 2024-11-22T19:45:16,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732304714507, region=ffec03947c9b817791870e0129146756, REOPEN/MOVE in 522 msec 2024-11-22T19:45:16,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T19:45:16,895 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T19:45:16,895 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T19:45:16,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:16,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:16,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T19:45:16,901 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T19:45:16,901 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-22T19:45:16,901 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=982bb8edae1e,39163,1732304711532 2024-11-22T19:45:16,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:16,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:16,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:16,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:16,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:16,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:16,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T19:45:16,904 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T19:45:16,904 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T19:45:16,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:16,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:16,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:16,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:16,905 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '982bb8edae1e,39485,1732304712304' ***** 2024-11-22T19:45:16,905 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T19:45:16,905 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '982bb8edae1e,44433,1732304712453' ***** 2024-11-22T19:45:16,905 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T19:45:16,905 INFO [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T19:45:16,905 INFO [RS:1;982bb8edae1e:44433 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T19:45:16,905 INFO [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.persistence 2024-11-22T19:45:16,905 INFO [RS:1;982bb8edae1e:44433 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/bucket.persistence 2024-11-22T19:45:16,906 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T19:45:16,906 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T19:45:16,906 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T19:45:16,906 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T19:45:16,906 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T19:45:16,906 INFO [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T19:45:16,906 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T19:45:16,906 INFO [RS:1;982bb8edae1e:44433 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T19:45:16,906 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T19:45:16,906 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T19:45:16,916 DEBUG [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1487): Thread RS:0;982bb8edae1e:39485 started persisting bucket cache to file 2024-11-22T19:45:16,916 DEBUG [RS:1;982bb8edae1e:44433 {}] bucket.BucketCache(1487): Thread RS:1;982bb8edae1e:44433 started persisting bucket cache to file 2024-11-22T19:45:16,916 DEBUG [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T19:45:16,916 DEBUG [RS:1;982bb8edae1e:44433 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T19:45:16,916 DEBUG [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T19:45:16,916 DEBUG [RS:1;982bb8edae1e:44433 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T19:45:16,920 DEBUG [RS:1;982bb8edae1e:44433 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T19:45:16,920 DEBUG [RS:1;982bb8edae1e:44433 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732304716916 2024-11-22T19:45:16,920 DEBUG [RS:1;982bb8edae1e:44433 {}] bucket.BucketCache(1509): Thread RS:1;982bb8edae1e:44433 finished persisting bucket cache to file, renaming 2024-11-22T19:45:16,920 DEBUG [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T19:45:16,920 DEBUG [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732304716916 2024-11-22T19:45:16,920 DEBUG [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1509): Thread RS:0;982bb8edae1e:39485 finished persisting bucket cache to file, renaming 2024-11-22T19:45:16,920 WARN [RS:0;982bb8edae1e:39485 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-22T19:45:16,920 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T19:45:16,920 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T19:45:16,921 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T19:45:16,921 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T19:45:16,921 INFO [RS:1;982bb8edae1e:44433 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T19:45:16,921 INFO [RS:0;982bb8edae1e:39485 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T19:45:16,921 INFO [RS:0;982bb8edae1e:39485 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T19:45:16,921 INFO [RS:1;982bb8edae1e:44433 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T19:45:16,921 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(959): stopping server 982bb8edae1e,39485,1732304712304 2024-11-22T19:45:16,921 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(3091): Received CLOSE for ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,921 INFO [RS:0;982bb8edae1e:39485 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T19:45:16,921 INFO [RS:0;982bb8edae1e:39485 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;982bb8edae1e:39485. 2024-11-22T19:45:16,921 DEBUG [RS:0;982bb8edae1e:39485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T19:45:16,921 DEBUG [RS:0;982bb8edae1e:39485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:16,922 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(976): stopping server 982bb8edae1e,39485,1732304712304; all regions closed. 2024-11-22T19:45:16,922 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(959): stopping server 982bb8edae1e,44433,1732304712453 2024-11-22T19:45:16,922 INFO [RS:1;982bb8edae1e:44433 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T19:45:16,922 INFO [RS:1;982bb8edae1e:44433 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;982bb8edae1e:44433. 2024-11-22T19:45:16,922 DEBUG [RS:1;982bb8edae1e:44433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T19:45:16,922 DEBUG [RS:1;982bb8edae1e:44433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:16,922 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T19:45:16,922 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T19:45:16,922 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T19:45:16,922 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T19:45:16,922 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T19:45:16,922 INFO [regionserver/982bb8edae1e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:16,922 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ffec03947c9b817791870e0129146756, disabling compactions & flushes 2024-11-22T19:45:16,923 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,923 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-22T19:45:16,923 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,923 DEBUG [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ffec03947c9b817791870e0129146756=testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756.} 2024-11-22T19:45:16,923 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T19:45:16,923 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. after waiting 0 ms 2024-11-22T19:45:16,923 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,923 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T19:45:16,923 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T19:45:16,923 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T19:45:16,923 DEBUG [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ffec03947c9b817791870e0129146756 2024-11-22T19:45:16,923 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T19:45:16,923 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T19:45:16,923 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-22T19:45:16,924 DEBUG [StoreFileCloser-ffec03947c9b817791870e0129146756-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/family/a072ec1126d24efe82bdf1744dea8ea3 2024-11-22T19:45:16,927 DEBUG [RS:0;982bb8edae1e:39485 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs 2024-11-22T19:45:16,927 INFO [RS:0;982bb8edae1e:39485 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 982bb8edae1e%2C39485%2C1732304712304:(num 1732304713763) 2024-11-22T19:45:16,927 DEBUG [RS:0;982bb8edae1e:39485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:16,927 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:16,928 INFO [RS:0;982bb8edae1e:39485 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T19:45:16,928 INFO [RS:0;982bb8edae1e:39485 {}] hbase.ChoreService(370): Chore service for: regionserver/982bb8edae1e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T19:45:16,928 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T19:45:16,928 INFO [regionserver/982bb8edae1e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T19:45:16,928 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T19:45:16,928 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T19:45:16,928 INFO [RS:0;982bb8edae1e:39485 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T19:45:16,929 INFO [RS:0;982bb8edae1e:39485 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39485 2024-11-22T19:45:16,931 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnRegionMove1732304714507/ffec03947c9b817791870e0129146756/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T19:45:16,932 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,932 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ffec03947c9b817791870e0129146756: Waiting for close lock at 1732304716922Running coprocessor pre-close hooks at 1732304716922Disabling compacts and flushes for region at 1732304716922Disabling writes for close at 1732304716923 (+1 ms)Writing region close event to WAL at 1732304716924 (+1 ms)Running coprocessor post-close hooks at 1732304716932 (+8 ms)Closed at 1732304716932 2024-11-22T19:45:16,932 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756. 2024-11-22T19:45:16,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/982bb8edae1e,39485,1732304712304 2024-11-22T19:45:16,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:16,933 INFO [RS:0;982bb8edae1e:39485 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T19:45:16,936 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [982bb8edae1e,39485,1732304712304] 2024-11-22T19:45:16,937 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 066e34fa244d46c982d6f31cdea5b770 2024-11-22T19:45:16,939 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/982bb8edae1e,39485,1732304712304 already deleted, retry=false 2024-11-22T19:45:16,939 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 982bb8edae1e,39485,1732304712304 expired; onlineServers=1 2024-11-22T19:45:16,955 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/info/066e34fa244d46c982d6f31cdea5b770 is 205, key is testBlockEvictionOnRegionMove1732304714507,,1732304714526.ffec03947c9b817791870e0129146756./info:regioninfo/1732304716856/Put/seqid=0 2024-11-22T19:45:16,956 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/info/066e34fa244d46c982d6f31cdea5b770 2024-11-22T19:45:16,965 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 158c4c5e19224330b047d3d573e65b59 2024-11-22T19:45:16,966 INFO [regionserver/982bb8edae1e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:16,979 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/ns/158c4c5e19224330b047d3d573e65b59 is 43, key is default/ns:d/1732304714316/Put/seqid=0 2024-11-22T19:45:16,980 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/ns/158c4c5e19224330b047d3d573e65b59 2024-11-22T19:45:16,988 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2653dce5e4054b1fb18e12b3d984a316 2024-11-22T19:45:17,001 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/table/2653dce5e4054b1fb18e12b3d984a316 is 78, key is testBlockEvictionOnRegionMove1732304714507/table:state/1732304714938/Put/seqid=0 2024-11-22T19:45:17,002 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/table/2653dce5e4054b1fb18e12b3d984a316 2024-11-22T19:45:17,004 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/info/066e34fa244d46c982d6f31cdea5b770 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/info/066e34fa244d46c982d6f31cdea5b770 2024-11-22T19:45:17,006 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/info/066e34fa244d46c982d6f31cdea5b770, delay=987 ms 2024-11-22T19:45:17,007 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/info/066e34fa244d46c982d6f31cdea5b770, entries=15, sequenceid=15, filesize=7.6 K 2024-11-22T19:45:17,007 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/ns/158c4c5e19224330b047d3d573e65b59 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/ns/158c4c5e19224330b047d3d573e65b59 2024-11-22T19:45:17,009 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/ns/158c4c5e19224330b047d3d573e65b59, delay=992 ms 2024-11-22T19:45:17,010 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/ns/158c4c5e19224330b047d3d573e65b59, entries=2, sequenceid=15, filesize=5.0 K 2024-11-22T19:45:17,010 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/table/2653dce5e4054b1fb18e12b3d984a316 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/table/2653dce5e4054b1fb18e12b3d984a316 2024-11-22T19:45:17,012 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/table/2653dce5e4054b1fb18e12b3d984a316, delay=915 ms 2024-11-22T19:45:17,013 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/table/2653dce5e4054b1fb18e12b3d984a316, entries=2, sequenceid=15, filesize=5.3 K 2024-11-22T19:45:17,014 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false 2024-11-22T19:45:17,014 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T19:45:17,015 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/info/066e34fa244d46c982d6f31cdea5b770 2024-11-22T19:45:17,016 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/ns/158c4c5e19224330b047d3d573e65b59 2024-11-22T19:45:17,016 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/table/2653dce5e4054b1fb18e12b3d984a316 2024-11-22T19:45:17,021 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-22T19:45:17,021 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T19:45:17,022 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T19:45:17,022 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732304716923Running coprocessor pre-close hooks at 1732304716923Disabling compacts and flushes for region at 1732304716923Disabling writes for close at 1732304716923Obtaining lock to block concurrent updates at 1732304716924 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732304716924Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732304716924Flushing stores of hbase:meta,,1.1588230740 at 1732304716924Flushing 1588230740/info: creating writer at 1732304716925 (+1 ms)Flushing 1588230740/info: appending metadata at 1732304716954 (+29 ms)Flushing 1588230740/info: closing flushed file at 1732304716954Flushing 1588230740/ns: creating writer at 1732304716958 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732304716978 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732304716978Flushing 1588230740/table: creating writer at 1732304716982 (+4 ms)Flushing 1588230740/table: appending metadata at 1732304717001 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732304717001Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@211963a7: reopening flushed file at 1732304717004 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@517a896b: reopening flushed file at 1732304717007 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e5303ea: reopening flushed file at 1732304717010 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false at 1732304717014 (+4 ms)Writing region close event to WAL at 1732304717017 (+3 ms)Running coprocessor post-close hooks at 1732304717021 (+4 ms)Closed at 1732304717022 (+1 ms) 2024-11-22T19:45:17,022 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T19:45:17,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:17,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39485-0x101413618b90001, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:17,037 INFO [RS:0;982bb8edae1e:39485 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T19:45:17,037 INFO [RS:0;982bb8edae1e:39485 {}] regionserver.HRegionServer(1031): Exiting; stopping=982bb8edae1e,39485,1732304712304; zookeeper connection closed. 2024-11-22T19:45:17,037 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28b697ba {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28b697ba 2024-11-22T19:45:17,123 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(976): stopping server 982bb8edae1e,44433,1732304712453; all regions closed. 2024-11-22T19:45:17,126 DEBUG [RS:1;982bb8edae1e:44433 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs 2024-11-22T19:45:17,126 INFO [RS:1;982bb8edae1e:44433 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 982bb8edae1e%2C44433%2C1732304712453.meta:.meta(num 1732304714192) 2024-11-22T19:45:17,128 DEBUG [RS:1;982bb8edae1e:44433 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs 2024-11-22T19:45:17,128 INFO [RS:1;982bb8edae1e:44433 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 982bb8edae1e%2C44433%2C1732304712453:(num 1732304713731) 2024-11-22T19:45:17,128 DEBUG [RS:1;982bb8edae1e:44433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:17,128 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:17,128 INFO [RS:1;982bb8edae1e:44433 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T19:45:17,128 INFO [RS:1;982bb8edae1e:44433 {}] hbase.ChoreService(370): Chore service for: regionserver/982bb8edae1e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T19:45:17,128 INFO [RS:1;982bb8edae1e:44433 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T19:45:17,129 INFO [regionserver/982bb8edae1e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T19:45:17,129 INFO [RS:1;982bb8edae1e:44433 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44433 2024-11-22T19:45:17,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:17,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/982bb8edae1e,44433,1732304712453 2024-11-22T19:45:17,131 INFO [RS:1;982bb8edae1e:44433 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T19:45:17,133 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [982bb8edae1e,44433,1732304712453] 2024-11-22T19:45:17,134 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/982bb8edae1e,44433,1732304712453 already deleted, retry=false 2024-11-22T19:45:17,134 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 982bb8edae1e,44433,1732304712453 expired; onlineServers=0 2024-11-22T19:45:17,135 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '982bb8edae1e,39163,1732304711532' ***** 2024-11-22T19:45:17,135 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T19:45:17,135 INFO [M:0;982bb8edae1e:39163 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T19:45:17,135 INFO [M:0;982bb8edae1e:39163 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T19:45:17,135 DEBUG [M:0;982bb8edae1e:39163 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T19:45:17,135 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T19:45:17,135 DEBUG [M:0;982bb8edae1e:39163 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T19:45:17,135 DEBUG [master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.small.0-1732304713451 {}] cleaner.HFileCleaner(306): Exit Thread[master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.small.0-1732304713451,5,FailOnTimeoutGroup] 2024-11-22T19:45:17,135 DEBUG [master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.large.0-1732304713450 {}] cleaner.HFileCleaner(306): Exit Thread[master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.large.0-1732304713450,5,FailOnTimeoutGroup] 2024-11-22T19:45:17,135 INFO [M:0;982bb8edae1e:39163 {}] hbase.ChoreService(370): Chore service for: master/982bb8edae1e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T19:45:17,136 INFO [M:0;982bb8edae1e:39163 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T19:45:17,136 DEBUG [M:0;982bb8edae1e:39163 {}] master.HMaster(1795): Stopping service threads 2024-11-22T19:45:17,136 INFO [M:0;982bb8edae1e:39163 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T19:45:17,136 INFO [M:0;982bb8edae1e:39163 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T19:45:17,137 INFO [M:0;982bb8edae1e:39163 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T19:45:17,137 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T19:45:17,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T19:45:17,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,138 DEBUG [M:0;982bb8edae1e:39163 {}] zookeeper.ZKUtil(347): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T19:45:17,138 WARN [M:0;982bb8edae1e:39163 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T19:45:17,139 INFO [M:0;982bb8edae1e:39163 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/.lastflushedseqids 2024-11-22T19:45:17,151 INFO [M:0;982bb8edae1e:39163 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T19:45:17,152 INFO [M:0;982bb8edae1e:39163 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T19:45:17,152 DEBUG [M:0;982bb8edae1e:39163 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T19:45:17,152 INFO [M:0;982bb8edae1e:39163 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:17,152 DEBUG [M:0;982bb8edae1e:39163 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:17,152 DEBUG [M:0;982bb8edae1e:39163 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T19:45:17,152 DEBUG [M:0;982bb8edae1e:39163 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:17,152 INFO [M:0;982bb8edae1e:39163 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-22T19:45:17,162 DEBUG [M:0;982bb8edae1e:39163 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 76c3e27a2ffc4686aae0510038bb565d 2024-11-22T19:45:17,176 DEBUG [M:0;982bb8edae1e:39163 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/76c3e27a2ffc4686aae0510038bb565d is 82, key is hbase:meta,,1/info:regioninfo/1732304714256/Put/seqid=0 2024-11-22T19:45:17,176 INFO [M:0;982bb8edae1e:39163 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/76c3e27a2ffc4686aae0510038bb565d 2024-11-22T19:45:17,184 DEBUG [M:0;982bb8edae1e:39163 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2af48a10db8a4d8e80a702d2b45df98c 2024-11-22T19:45:17,198 DEBUG [M:0;982bb8edae1e:39163 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2af48a10db8a4d8e80a702d2b45df98c is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732304714943/Put/seqid=0 2024-11-22T19:45:17,198 INFO [M:0;982bb8edae1e:39163 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2af48a10db8a4d8e80a702d2b45df98c 2024-11-22T19:45:17,206 DEBUG [M:0;982bb8edae1e:39163 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on de5740b4007e4fd7b33087746713e245 2024-11-22T19:45:17,219 DEBUG [M:0;982bb8edae1e:39163 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de5740b4007e4fd7b33087746713e245 is 69, key is 982bb8edae1e,39485,1732304712304/rs:state/1732304713590/Put/seqid=0 2024-11-22T19:45:17,219 INFO [M:0;982bb8edae1e:39163 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de5740b4007e4fd7b33087746713e245 2024-11-22T19:45:17,221 DEBUG [M:0;982bb8edae1e:39163 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/76c3e27a2ffc4686aae0510038bb565d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/76c3e27a2ffc4686aae0510038bb565d 2024-11-22T19:45:17,223 INFO [M:0;982bb8edae1e:39163 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/76c3e27a2ffc4686aae0510038bb565d, entries=8, sequenceid=84, filesize=5.5 K 2024-11-22T19:45:17,223 DEBUG [M:0;982bb8edae1e:39163 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2af48a10db8a4d8e80a702d2b45df98c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2af48a10db8a4d8e80a702d2b45df98c 2024-11-22T19:45:17,225 INFO [M:0;982bb8edae1e:39163 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2af48a10db8a4d8e80a702d2b45df98c, entries=9, sequenceid=84, filesize=6.5 K 2024-11-22T19:45:17,225 DEBUG [M:0;982bb8edae1e:39163 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de5740b4007e4fd7b33087746713e245 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/de5740b4007e4fd7b33087746713e245 2024-11-22T19:45:17,226 INFO [M:0;982bb8edae1e:39163 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/de5740b4007e4fd7b33087746713e245, entries=2, sequenceid=84, filesize=5.1 K 2024-11-22T19:45:17,227 INFO [M:0;982bb8edae1e:39163 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=84, compaction requested=false 2024-11-22T19:45:17,229 INFO [M:0;982bb8edae1e:39163 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:17,229 DEBUG [M:0;982bb8edae1e:39163 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732304717152Disabling compacts and flushes for region at 1732304717152Disabling writes for close at 1732304717152Obtaining lock to block concurrent updates at 1732304717152Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732304717152Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1732304717153 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732304717153Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732304717153Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732304717175 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732304717175Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732304717178 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732304717197 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732304717197Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732304717200 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732304717218 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732304717218Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d10f2f: reopening flushed file at 1732304717221 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1732304717223 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1732304717225 (+2 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=84, compaction requested=false at 1732304717227 (+2 ms)Writing region close event to WAL at 1732304717229 (+2 ms)Closed at 1732304717229 2024-11-22T19:45:17,230 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T19:45:17,231 INFO [M:0;982bb8edae1e:39163 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T19:45:17,231 INFO [M:0;982bb8edae1e:39163 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39163 2024-11-22T19:45:17,231 INFO [M:0;982bb8edae1e:39163 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T19:45:17,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:17,233 INFO [RS:1;982bb8edae1e:44433 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T19:45:17,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44433-0x101413618b90002, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:17,233 INFO [RS:1;982bb8edae1e:44433 {}] regionserver.HRegionServer(1031): Exiting; stopping=982bb8edae1e,44433,1732304712453; zookeeper connection closed. 2024-11-22T19:45:17,233 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b6aab8f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b6aab8f 2024-11-22T19:45:17,234 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-22T19:45:17,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:17,333 INFO [M:0;982bb8edae1e:39163 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T19:45:17,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39163-0x101413618b90000, quorum=127.0.0.1:65249, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:17,338 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T19:45:17,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T19:45:17,357 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:17,366 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/982bb8edae1e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732304715318 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/982bb8edae1e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/982bb8edae1e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732304717009 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732304717006 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732304716843 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/982bb8edae1e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 276) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8745 (was 9117) 2024-11-22T19:45:17,372 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=8745 2024-11-22T19:45:17,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/hadoop-log-dir so I do NOT create it in target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885 2024-11-22T19:45:17,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/hadoop-tmp-dir so I do NOT create it in target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885 2024-11-22T19:45:17,373 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:17,376 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/cluster_0770be41-dce8-95a7-0a7e-ca39342a1cc3, deleteOnExit=true 2024-11-22T19:45:17,379 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/cluster_0770be41-dce8-95a7-0a7e-ca39342a1cc3/zookeeper_0, clientPort=53451, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/cluster_0770be41-dce8-95a7-0a7e-ca39342a1cc3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/cluster_0770be41-dce8-95a7-0a7e-ca39342a1cc3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T19:45:17,380 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53451 2024-11-22T19:45:17,392 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 with version=8 2024-11-22T19:45:17,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/hbase-staging 2024-11-22T19:45:17,394 INFO [Time-limited test {}] client.ConnectionUtils(128): master/982bb8edae1e:0 server-side Connection retries=45 2024-11-22T19:45:17,394 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:17,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:17,395 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T19:45:17,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:17,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T19:45:17,395 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T19:45:17,395 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T19:45:17,396 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36577 2024-11-22T19:45:17,397 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36577 connecting to ZooKeeper ensemble=127.0.0.1:53451 2024-11-22T19:45:17,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:365770x0, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T19:45:17,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36577-0x101413630d20000 connected 2024-11-22T19:45:17,420 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:17,420 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:17,421 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:17,421 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816, hbase.cluster.distributed=false 2024-11-22T19:45:17,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T19:45:17,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36577 2024-11-22T19:45:17,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36577 2024-11-22T19:45:17,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36577 2024-11-22T19:45:17,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36577 2024-11-22T19:45:17,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36577 2024-11-22T19:45:17,440 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/982bb8edae1e:0 server-side Connection retries=45 2024-11-22T19:45:17,441 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:17,441 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:17,441 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T19:45:17,441 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:17,441 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T19:45:17,441 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T19:45:17,441 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T19:45:17,442 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35577 2024-11-22T19:45:17,443 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35577 connecting to ZooKeeper ensemble=127.0.0.1:53451 2024-11-22T19:45:17,443 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:17,443 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:17,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355770x0, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T19:45:17,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35577-0x101413630d20001 connected 2024-11-22T19:45:17,447 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:17,447 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T19:45:17,448 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.cache 2024-11-22T19:45:17,448 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T19:45:17,449 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T19:45:17,449 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T19:45:17,449 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T19:45:17,449 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T19:45:17,449 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T19:45:17,450 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T19:45:17,450 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.persistence is complete. 2024-11-22T19:45:17,451 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T19:45:17,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T19:45:17,453 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T19:45:17,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35577 2024-11-22T19:45:17,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35577 2024-11-22T19:45:17,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35577 2024-11-22T19:45:17,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35577 2024-11-22T19:45:17,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35577 2024-11-22T19:45:17,474 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/982bb8edae1e:0 server-side Connection retries=45 2024-11-22T19:45:17,474 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:17,474 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:17,474 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T19:45:17,474 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:17,474 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T19:45:17,475 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T19:45:17,475 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T19:45:17,475 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35919 2024-11-22T19:45:17,477 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35919 connecting to ZooKeeper ensemble=127.0.0.1:53451 2024-11-22T19:45:17,477 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:17,477 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:17,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359190x0, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T19:45:17,480 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35919-0x101413630d20002 connected 2024-11-22T19:45:17,480 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:17,481 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T19:45:17,481 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.cache 2024-11-22T19:45:17,482 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T19:45:17,482 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T19:45:17,482 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T19:45:17,482 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T19:45:17,482 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T19:45:17,482 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T19:45:17,482 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T19:45:17,482 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.persistence is complete. 2024-11-22T19:45:17,483 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T19:45:17,484 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T19:45:17,485 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T19:45:17,487 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35919 2024-11-22T19:45:17,487 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35919 2024-11-22T19:45:17,488 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35919 2024-11-22T19:45:17,489 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35919 2024-11-22T19:45:17,489 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35919 2024-11-22T19:45:17,502 DEBUG [M:0;982bb8edae1e:36577 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;982bb8edae1e:36577 2024-11-22T19:45:17,502 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/982bb8edae1e,36577,1732304717394 2024-11-22T19:45:17,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:17,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:17,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:17,506 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/982bb8edae1e,36577,1732304717394 2024-11-22T19:45:17,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T19:45:17,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T19:45:17,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,508 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T19:45:17,511 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/982bb8edae1e,36577,1732304717394 from backup master directory 2024-11-22T19:45:17,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/982bb8edae1e,36577,1732304717394 2024-11-22T19:45:17,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:17,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:17,512 WARN [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T19:45:17,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T19:45:17,512 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=982bb8edae1e,36577,1732304717394 2024-11-22T19:45:17,514 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/hbase.id] with ID: 84354ac9-86be-4d4f-8ec5-886fc7879516 2024-11-22T19:45:17,514 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/.tmp/hbase.id 2024-11-22T19:45:17,520 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/hbase.id] 2024-11-22T19:45:17,559 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:17,559 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T19:45:17,561 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-22T19:45:17,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,570 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T19:45:17,572 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T19:45:17,573 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T19:45:17,593 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store 2024-11-22T19:45:17,599 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:17,599 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T19:45:17,599 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:17,599 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:17,599 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T19:45:17,599 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:17,599 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:17,599 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732304717599Disabling compacts and flushes for region at 1732304717599Disabling writes for close at 1732304717599Writing region close event to WAL at 1732304717599Closed at 1732304717599 2024-11-22T19:45:17,599 WARN [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/.initializing 2024-11-22T19:45:17,600 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/WALs/982bb8edae1e,36577,1732304717394 2024-11-22T19:45:17,605 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T19:45:17,608 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=982bb8edae1e%2C36577%2C1732304717394, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/WALs/982bb8edae1e,36577,1732304717394, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/oldWALs, maxLogs=10 2024-11-22T19:45:17,626 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/WALs/982bb8edae1e,36577,1732304717394/982bb8edae1e%2C36577%2C1732304717394.1732304717609 2024-11-22T19:45:17,627 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T19:45:17,627 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T19:45:17,627 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:17,627 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,627 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T19:45:17,634 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:17,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:17,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T19:45:17,638 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:17,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:17,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T19:45:17,642 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:17,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:17,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T19:45:17,646 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:17,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:17,647 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,647 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,647 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,647 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,647 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,648 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T19:45:17,648 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T19:45:17,654 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T19:45:17,655 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63626482, jitterRate=-0.05189153552055359}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T19:45:17,656 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732304717627Initializing all the Stores at 1732304717627Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304717628 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304717628Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304717628Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304717628Cleaning up temporary data from old regions at 1732304717647 (+19 ms)Region opened successfully at 1732304717656 (+9 ms) 2024-11-22T19:45:17,657 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T19:45:17,660 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4880066d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=982bb8edae1e/172.17.0.2:0 2024-11-22T19:45:17,662 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T19:45:17,662 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T19:45:17,662 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T19:45:17,662 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T19:45:17,662 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T19:45:17,663 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-22T19:45:17,663 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T19:45:17,664 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T19:45:17,665 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T19:45:17,668 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T19:45:17,669 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T19:45:17,669 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T19:45:17,670 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T19:45:17,671 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T19:45:17,672 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T19:45:17,673 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T19:45:17,674 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T19:45:17,675 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T19:45:17,677 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T19:45:17,679 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T19:45:17,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:17,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:17,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:17,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,682 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=982bb8edae1e,36577,1732304717394, sessionid=0x101413630d20000, setting cluster-up flag (Was=false) 2024-11-22T19:45:17,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,692 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T19:45:17,693 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=982bb8edae1e,36577,1732304717394 2024-11-22T19:45:17,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:17,703 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T19:45:17,705 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=982bb8edae1e,36577,1732304717394 2024-11-22T19:45:17,705 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T19:45:17,707 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T19:45:17,707 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T19:45:17,707 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T19:45:17,708 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 982bb8edae1e,36577,1732304717394 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T19:45:17,709 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/982bb8edae1e:0, corePoolSize=5, maxPoolSize=5 2024-11-22T19:45:17,710 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/982bb8edae1e:0, corePoolSize=5, maxPoolSize=5 2024-11-22T19:45:17,710 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/982bb8edae1e:0, corePoolSize=5, maxPoolSize=5 2024-11-22T19:45:17,710 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/982bb8edae1e:0, corePoolSize=5, maxPoolSize=5 2024-11-22T19:45:17,710 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/982bb8edae1e:0, corePoolSize=10, maxPoolSize=10 2024-11-22T19:45:17,710 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,710 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/982bb8edae1e:0, corePoolSize=2, maxPoolSize=2 2024-11-22T19:45:17,711 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,711 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732304747711 2024-11-22T19:45:17,712 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T19:45:17,712 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T19:45:17,712 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T19:45:17,712 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T19:45:17,712 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T19:45:17,712 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T19:45:17,712 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T19:45:17,712 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T19:45:17,713 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,713 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:17,713 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T19:45:17,713 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T19:45:17,714 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T19:45:17,714 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T19:45:17,715 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T19:45:17,715 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T19:45:17,715 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.large.0-1732304717715,5,FailOnTimeoutGroup] 2024-11-22T19:45:17,715 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.small.0-1732304717715,5,FailOnTimeoutGroup] 2024-11-22T19:45:17,715 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,716 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T19:45:17,716 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,716 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,736 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T19:45:17,736 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 2024-11-22T19:45:17,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:17,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T19:45:17,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T19:45:17,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:17,752 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:17,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T19:45:17,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T19:45:17,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:17,757 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:17,757 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T19:45:17,761 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T19:45:17,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:17,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:17,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T19:45:17,765 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T19:45:17,765 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:17,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:17,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T19:45:17,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740 2024-11-22T19:45:17,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740 2024-11-22T19:45:17,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T19:45:17,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T19:45:17,767 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T19:45:17,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T19:45:17,773 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T19:45:17,773 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63784974, jitterRate=-0.049529820680618286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T19:45:17,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732304717744Initializing all the Stores at 1732304717745 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304717745Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304717745Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304717745Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304717745Cleaning up temporary data from old regions at 1732304717766 (+21 ms)Region opened successfully at 1732304717774 (+8 ms) 2024-11-22T19:45:17,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T19:45:17,774 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T19:45:17,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T19:45:17,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T19:45:17,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T19:45:17,775 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T19:45:17,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732304717774Disabling compacts and flushes for region at 1732304717774Disabling writes for close at 1732304717774Writing region close event to WAL at 1732304717775 (+1 ms)Closed at 1732304717775 2024-11-22T19:45:17,776 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T19:45:17,776 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T19:45:17,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T19:45:17,777 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T19:45:17,778 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T19:45:17,793 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(746): ClusterId : 84354ac9-86be-4d4f-8ec5-886fc7879516 2024-11-22T19:45:17,793 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(746): ClusterId : 84354ac9-86be-4d4f-8ec5-886fc7879516 2024-11-22T19:45:17,793 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T19:45:17,793 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T19:45:17,796 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T19:45:17,796 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T19:45:17,796 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T19:45:17,796 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T19:45:17,799 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T19:45:17,799 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T19:45:17,799 DEBUG [RS:1;982bb8edae1e:35919 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4538eb92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=982bb8edae1e/172.17.0.2:0 2024-11-22T19:45:17,800 DEBUG [RS:0;982bb8edae1e:35577 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f028b2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=982bb8edae1e/172.17.0.2:0 2024-11-22T19:45:17,813 DEBUG [RS:0;982bb8edae1e:35577 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;982bb8edae1e:35577 2024-11-22T19:45:17,813 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T19:45:17,813 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T19:45:17,813 DEBUG [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T19:45:17,814 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(2659): reportForDuty to master=982bb8edae1e,36577,1732304717394 with port=35577, startcode=1732304717440 2024-11-22T19:45:17,814 DEBUG [RS:0;982bb8edae1e:35577 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T19:45:17,817 DEBUG [RS:1;982bb8edae1e:35919 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;982bb8edae1e:35919 2024-11-22T19:45:17,817 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T19:45:17,817 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T19:45:17,817 DEBUG [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T19:45:17,817 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42229, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T19:45:17,818 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 982bb8edae1e,35577,1732304717440 2024-11-22T19:45:17,818 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(2659): reportForDuty to master=982bb8edae1e,36577,1732304717394 with port=35919, startcode=1732304717474 2024-11-22T19:45:17,818 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36577 {}] master.ServerManager(517): Registering regionserver=982bb8edae1e,35577,1732304717440 2024-11-22T19:45:17,818 DEBUG [RS:1;982bb8edae1e:35919 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T19:45:17,820 DEBUG [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 2024-11-22T19:45:17,820 DEBUG [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T19:45:17,820 DEBUG [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T19:45:17,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:17,823 DEBUG [RS:0;982bb8edae1e:35577 {}] zookeeper.ZKUtil(111): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/982bb8edae1e,35577,1732304717440 2024-11-22T19:45:17,823 WARN [RS:0;982bb8edae1e:35577 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T19:45:17,823 INFO [RS:0;982bb8edae1e:35577 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T19:45:17,823 DEBUG [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35577,1732304717440 2024-11-22T19:45:17,825 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48867, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T19:45:17,826 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [982bb8edae1e,35577,1732304717440] 2024-11-22T19:45:17,826 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 982bb8edae1e,35919,1732304717474 2024-11-22T19:45:17,826 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36577 {}] master.ServerManager(517): Registering regionserver=982bb8edae1e,35919,1732304717474 2024-11-22T19:45:17,828 DEBUG [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 2024-11-22T19:45:17,828 DEBUG [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T19:45:17,828 DEBUG [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T19:45:17,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:17,833 DEBUG [RS:1;982bb8edae1e:35919 {}] zookeeper.ZKUtil(111): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/982bb8edae1e,35919,1732304717474 2024-11-22T19:45:17,834 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T19:45:17,834 WARN [RS:1;982bb8edae1e:35919 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T19:45:17,834 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [982bb8edae1e,35919,1732304717474] 2024-11-22T19:45:17,834 INFO [RS:1;982bb8edae1e:35919 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T19:45:17,834 DEBUG [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35919,1732304717474 2024-11-22T19:45:17,835 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T19:45:17,836 INFO [RS:0;982bb8edae1e:35577 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T19:45:17,836 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,836 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T19:45:17,837 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T19:45:17,837 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,837 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,837 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,837 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,837 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,837 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,837 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/982bb8edae1e:0, corePoolSize=2, maxPoolSize=2 2024-11-22T19:45:17,837 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,838 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,838 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,838 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,838 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,838 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,838 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:17,838 DEBUG [RS:0;982bb8edae1e:35577 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:17,839 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,839 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,839 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,839 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,839 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,839 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,35577,1732304717440-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T19:45:17,840 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T19:45:17,845 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T19:45:17,846 INFO [RS:1;982bb8edae1e:35919 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T19:45:17,846 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,846 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T19:45:17,847 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T19:45:17,847 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,847 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,847 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/982bb8edae1e:0, corePoolSize=2, maxPoolSize=2 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,848 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:17,849 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:17,849 DEBUG [RS:1;982bb8edae1e:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:17,849 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,849 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,849 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,849 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,849 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,849 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,35919,1732304717474-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T19:45:17,855 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T19:45:17,855 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,35577,1732304717440-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,856 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,856 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.Replication(171): 982bb8edae1e,35577,1732304717440 started 2024-11-22T19:45:17,866 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T19:45:17,866 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,35919,1732304717474-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,866 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,866 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.Replication(171): 982bb8edae1e,35919,1732304717474 started 2024-11-22T19:45:17,871 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,871 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(1482): Serving as 982bb8edae1e,35577,1732304717440, RpcServer on 982bb8edae1e/172.17.0.2:35577, sessionid=0x101413630d20001 2024-11-22T19:45:17,871 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T19:45:17,871 DEBUG [RS:0;982bb8edae1e:35577 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 982bb8edae1e,35577,1732304717440 2024-11-22T19:45:17,871 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,35577,1732304717440' 2024-11-22T19:45:17,871 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T19:45:17,872 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T19:45:17,873 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T19:45:17,873 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T19:45:17,873 DEBUG [RS:0;982bb8edae1e:35577 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 982bb8edae1e,35577,1732304717440 2024-11-22T19:45:17,873 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,35577,1732304717440' 2024-11-22T19:45:17,873 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T19:45:17,873 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T19:45:17,874 DEBUG [RS:0;982bb8edae1e:35577 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T19:45:17,874 INFO [RS:0;982bb8edae1e:35577 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T19:45:17,874 INFO [RS:0;982bb8edae1e:35577 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T19:45:17,881 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:17,882 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(1482): Serving as 982bb8edae1e,35919,1732304717474, RpcServer on 982bb8edae1e/172.17.0.2:35919, sessionid=0x101413630d20002 2024-11-22T19:45:17,882 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T19:45:17,882 DEBUG [RS:1;982bb8edae1e:35919 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 982bb8edae1e,35919,1732304717474 2024-11-22T19:45:17,882 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,35919,1732304717474' 2024-11-22T19:45:17,882 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T19:45:17,882 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T19:45:17,883 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T19:45:17,883 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T19:45:17,883 DEBUG [RS:1;982bb8edae1e:35919 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 982bb8edae1e,35919,1732304717474 2024-11-22T19:45:17,883 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,35919,1732304717474' 2024-11-22T19:45:17,883 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T19:45:17,883 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T19:45:17,884 DEBUG [RS:1;982bb8edae1e:35919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T19:45:17,884 INFO [RS:1;982bb8edae1e:35919 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T19:45:17,884 INFO [RS:1;982bb8edae1e:35919 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T19:45:17,928 WARN [982bb8edae1e:36577 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T19:45:17,974 INFO [RS:0;982bb8edae1e:35577 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T19:45:17,976 INFO [RS:0;982bb8edae1e:35577 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=982bb8edae1e%2C35577%2C1732304717440, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35577,1732304717440, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs, maxLogs=55 2024-11-22T19:45:17,984 INFO [RS:1;982bb8edae1e:35919 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T19:45:17,985 INFO [RS:1;982bb8edae1e:35919 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=982bb8edae1e%2C35919%2C1732304717474, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35919,1732304717474, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs, maxLogs=55 2024-11-22T19:45:17,995 INFO [RS:0;982bb8edae1e:35577 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35577,1732304717440/982bb8edae1e%2C35577%2C1732304717440.1732304717978 2024-11-22T19:45:17,996 DEBUG [RS:0;982bb8edae1e:35577 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T19:45:18,009 INFO [RS:1;982bb8edae1e:35919 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35919,1732304717474/982bb8edae1e%2C35919%2C1732304717474.1732304717986 2024-11-22T19:45:18,010 DEBUG [RS:1;982bb8edae1e:35919 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T19:45:18,178 DEBUG [982bb8edae1e:36577 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T19:45:18,179 DEBUG [982bb8edae1e:36577 {}] balancer.BalancerClusterState(204): Hosts are {982bb8edae1e=0} racks are {/default-rack=0} 2024-11-22T19:45:18,179 DEBUG [982bb8edae1e:36577 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T19:45:18,179 DEBUG [982bb8edae1e:36577 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T19:45:18,179 DEBUG [982bb8edae1e:36577 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T19:45:18,179 DEBUG [982bb8edae1e:36577 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T19:45:18,179 INFO [982bb8edae1e:36577 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T19:45:18,179 INFO [982bb8edae1e:36577 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T19:45:18,179 DEBUG [982bb8edae1e:36577 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T19:45:18,180 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=982bb8edae1e,35919,1732304717474 2024-11-22T19:45:18,181 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 982bb8edae1e,35919,1732304717474, state=OPENING 2024-11-22T19:45:18,183 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T19:45:18,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:18,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:18,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:18,185 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T19:45:18,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:18,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:18,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:18,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=982bb8edae1e,35919,1732304717474}] 2024-11-22T19:45:18,338 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T19:45:18,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34869, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T19:45:18,342 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T19:45:18,343 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T19:45:18,343 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T19:45:18,345 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=982bb8edae1e%2C35919%2C1732304717474.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35919,1732304717474, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs, maxLogs=55 2024-11-22T19:45:18,364 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35919,1732304717474/982bb8edae1e%2C35919%2C1732304717474.meta.1732304718345.meta 2024-11-22T19:45:18,364 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T19:45:18,365 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T19:45:18,365 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T19:45:18,365 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T19:45:18,365 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T19:45:18,365 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T19:45:18,366 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:18,366 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T19:45:18,366 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T19:45:18,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T19:45:18,368 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T19:45:18,368 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:18,368 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:18,368 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T19:45:18,369 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T19:45:18,369 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:18,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:18,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T19:45:18,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T19:45:18,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:18,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:18,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T19:45:18,371 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T19:45:18,371 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:18,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T19:45:18,371 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T19:45:18,372 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740 2024-11-22T19:45:18,372 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740 2024-11-22T19:45:18,372 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T19:45:18,372 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T19:45:18,373 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T19:45:18,373 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T19:45:18,374 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71146356, jitterRate=0.06016331911087036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T19:45:18,374 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T19:45:18,375 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732304718366Writing region info on filesystem at 1732304718366Initializing all the Stores at 1732304718366Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304718366Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304718366Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304718366Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732304718366Cleaning up temporary data from old regions at 1732304718372 (+6 ms)Running coprocessor post-open hooks at 1732304718374 (+2 ms)Region opened successfully at 1732304718375 (+1 ms) 2024-11-22T19:45:18,375 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732304718337 2024-11-22T19:45:18,378 DEBUG [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T19:45:18,378 INFO [RS_OPEN_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T19:45:18,379 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=982bb8edae1e,35919,1732304717474 2024-11-22T19:45:18,380 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 982bb8edae1e,35919,1732304717474, state=OPEN 2024-11-22T19:45:18,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T19:45:18,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T19:45:18,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T19:45:18,382 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=982bb8edae1e,35919,1732304717474 2024-11-22T19:45:18,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:18,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:18,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T19:45:18,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T19:45:18,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=982bb8edae1e,35919,1732304717474 in 197 msec 2024-11-22T19:45:18,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T19:45:18,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-11-22T19:45:18,386 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T19:45:18,386 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T19:45:18,389 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T19:45:18,389 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=982bb8edae1e,35919,1732304717474, seqNum=-1] 2024-11-22T19:45:18,390 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T19:45:18,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39199, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T19:45:18,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 690 msec 2024-11-22T19:45:18,398 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732304718397, completionTime=-1 2024-11-22T19:45:18,398 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T19:45:18,398 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T19:45:18,400 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T19:45:18,400 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732304778400 2024-11-22T19:45:18,400 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732304838400 2024-11-22T19:45:18,400 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-22T19:45:18,401 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,36577,1732304717394-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:18,401 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,36577,1732304717394-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:18,401 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,36577,1732304717394-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:18,401 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-982bb8edae1e:36577, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:18,401 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:18,401 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:18,404 DEBUG [master/982bb8edae1e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T19:45:18,411 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.899sec 2024-11-22T19:45:18,411 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T19:45:18,411 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T19:45:18,411 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T19:45:18,411 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T19:45:18,411 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T19:45:18,411 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,36577,1732304717394-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T19:45:18,411 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,36577,1732304717394-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T19:45:18,412 DEBUG [master/982bb8edae1e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T19:45:18,412 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T19:45:18,413 INFO [master/982bb8edae1e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,36577,1732304717394-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:18,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T19:45:18,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 982bb8edae1e,36577,-1 for getting cluster id 2024-11-22T19:45:18,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T19:45:18,495 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84354ac9-86be-4d4f-8ec5-886fc7879516' 2024-11-22T19:45:18,496 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T19:45:18,496 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84354ac9-86be-4d4f-8ec5-886fc7879516" 2024-11-22T19:45:18,496 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc9c4be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T19:45:18,496 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [982bb8edae1e,36577,-1] 2024-11-22T19:45:18,496 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T19:45:18,497 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:18,498 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T19:45:18,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695e6613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T19:45:18,500 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T19:45:18,502 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=982bb8edae1e,35919,1732304717474, seqNum=-1] 2024-11-22T19:45:18,502 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T19:45:18,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T19:45:18,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=982bb8edae1e,36577,1732304717394 2024-11-22T19:45:18,507 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T19:45:18,508 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 982bb8edae1e,36577,1732304717394 2024-11-22T19:45:18,508 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@444123b9 2024-11-22T19:45:18,508 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T19:45:18,510 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T19:45:18,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732304718506', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T19:45:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36577 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732304718506 2024-11-22T19:45:18,513 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732304718506 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T19:45:18,513 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:18,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732304718506" procId is: 4 2024-11-22T19:45:18,514 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732304718506 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T19:45:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T19:45:18,525 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732304718506-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 83f9081ac3d35c2353cfcd3ee4517925, NAME => 'testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732304718506', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 2024-11-22T19:45:18,532 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732304718506-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:18,532 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732304718506-pool-0 {}] regionserver.HRegion(1722): Closing 83f9081ac3d35c2353cfcd3ee4517925, disabling compactions & flushes 2024-11-22T19:45:18,532 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732304718506-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:18,532 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732304718506-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:18,532 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732304718506-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. after waiting 0 ms 2024-11-22T19:45:18,533 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732304718506-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:18,533 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732304718506-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:18,533 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732304718506-pool-0 {}] regionserver.HRegion(1676): Region close journal for 83f9081ac3d35c2353cfcd3ee4517925: Waiting for close lock at 1732304718532Disabling compacts and flushes for region at 1732304718532Disabling writes for close at 1732304718532Writing region close event to WAL at 1732304718533 (+1 ms)Closed at 1732304718533 2024-11-22T19:45:18,534 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732304718506 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T19:45:18,534 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732304718534"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732304718534"}]},"ts":"1732304718534"} 2024-11-22T19:45:18,537 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T19:45:18,538 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732304718506 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T19:45:18,538 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732304718506","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732304718538"}]},"ts":"1732304718538"} 2024-11-22T19:45:18,541 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732304718506, state=ENABLING in hbase:meta 2024-11-22T19:45:18,541 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {982bb8edae1e=0} racks are {/default-rack=0} 2024-11-22T19:45:18,541 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T19:45:18,541 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T19:45:18,541 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T19:45:18,541 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T19:45:18,541 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T19:45:18,542 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T19:45:18,542 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T19:45:18,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN}] 2024-11-22T19:45:18,543 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN 2024-11-22T19:45:18,544 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN; state=OFFLINE, location=982bb8edae1e,35577,1732304717440; forceNewPlan=false, retain=false 2024-11-22T19:45:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T19:45:18,694 INFO [982bb8edae1e:36577 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T19:45:18,695 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=83f9081ac3d35c2353cfcd3ee4517925, regionState=OPENING, regionLocation=982bb8edae1e,35577,1732304717440 2024-11-22T19:45:18,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN because future has completed 2024-11-22T19:45:18,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83f9081ac3d35c2353cfcd3ee4517925, server=982bb8edae1e,35577,1732304717440}] 2024-11-22T19:45:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T19:45:18,851 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T19:45:18,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50897, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T19:45:18,854 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:18,854 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 83f9081ac3d35c2353cfcd3ee4517925, NAME => 'testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925.', STARTKEY => '', ENDKEY => ''} 2024-11-22T19:45:18,855 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732304718506 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,855 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:18,855 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,855 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,856 INFO [StoreOpener-83f9081ac3d35c2353cfcd3ee4517925-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,860 INFO [StoreOpener-83f9081ac3d35c2353cfcd3ee4517925-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83f9081ac3d35c2353cfcd3ee4517925 columnFamilyName family 2024-11-22T19:45:18,860 DEBUG [StoreOpener-83f9081ac3d35c2353cfcd3ee4517925-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:18,860 INFO [StoreOpener-83f9081ac3d35c2353cfcd3ee4517925-1 {}] regionserver.HStore(327): Store=83f9081ac3d35c2353cfcd3ee4517925/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:18,860 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,861 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,861 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,861 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,861 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,862 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,868 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T19:45:18,868 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 83f9081ac3d35c2353cfcd3ee4517925; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74423010, jitterRate=0.10898926854133606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T19:45:18,869 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:18,869 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 83f9081ac3d35c2353cfcd3ee4517925: Running coprocessor pre-open hook at 1732304718855Writing region info on filesystem at 1732304718855Initializing all the Stores at 1732304718856 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304718856Cleaning up temporary data from old regions at 1732304718861 (+5 ms)Running coprocessor post-open hooks at 1732304718869 (+8 ms)Region opened successfully at 1732304718869 2024-11-22T19:45:18,870 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925., pid=6, masterSystemTime=1732304718850 2024-11-22T19:45:18,873 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:18,873 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:18,873 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=83f9081ac3d35c2353cfcd3ee4517925, regionState=OPEN, openSeqNum=2, regionLocation=982bb8edae1e,35577,1732304717440 2024-11-22T19:45:18,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83f9081ac3d35c2353cfcd3ee4517925, server=982bb8edae1e,35577,1732304717440 because future has completed 2024-11-22T19:45:18,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T19:45:18,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 83f9081ac3d35c2353cfcd3ee4517925, server=982bb8edae1e,35577,1732304717440 in 178 msec 2024-11-22T19:45:18,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T19:45:18,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN in 336 msec 2024-11-22T19:45:18,880 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732304718506 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T19:45:18,880 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732304718506","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732304718880"}]},"ts":"1732304718880"} 2024-11-22T19:45:18,882 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732304718506, state=ENABLED in hbase:meta 2024-11-22T19:45:18,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732304718506 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T19:45:18,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732304718506 in 372 msec 2024-11-22T19:45:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T19:45:19,142 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732304718506 get assigned. Timeout = 60000ms 2024-11-22T19:45:19,142 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732304718506 completed 2024-11-22T19:45:19,142 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T19:45:19,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732304718506 assigned to meta. Checking AM states. 2024-11-22T19:45:19,146 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T19:45:19,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732304718506 assigned. 2024-11-22T19:45:19,149 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732304718506', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925., hostname=982bb8edae1e,35577,1732304717440, seqNum=2] 2024-11-22T19:45:19,150 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T19:45:19,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T19:45:19,156 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 83f9081ac3d35c2353cfcd3ee4517925 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T19:45:19,166 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 00fc03c51b1f4448beb9d5b6b8881b49 2024-11-22T19:45:19,180 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/.tmp/family/00fc03c51b1f4448beb9d5b6b8881b49 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T19:45:19,180 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/.tmp/family/00fc03c51b1f4448beb9d5b6b8881b49 2024-11-22T19:45:19,183 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/.tmp/family/00fc03c51b1f4448beb9d5b6b8881b49 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49 2024-11-22T19:45:19,184 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49, delay=949 ms 2024-11-22T19:45:19,185 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T19:45:19,186 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 83f9081ac3d35c2353cfcd3ee4517925 in 30ms, sequenceid=6, compaction requested=false 2024-11-22T19:45:19,186 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732304718506' 2024-11-22T19:45:19,187 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 83f9081ac3d35c2353cfcd3ee4517925: 2024-11-22T19:45:19,727 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T19:45:20,134 DEBUG [hfile-prefetch-1732304716843 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 00fc03c51b1f4448beb9d5b6b8881b49_0. 2024-11-22T19:45:20,135 DEBUG [hfile-prefetch-1732304716843 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 00fc03c51b1f4448beb9d5b6b8881b49_117. 2024-11-22T19:45:20,135 DEBUG [hfile-prefetch-1732304716843 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49, with total blocks 2, and data blocks 1 2024-11-22T19:45:20,135 DEBUG [hfile-prefetch-1732304716843 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T19:45:20,135 DEBUG [hfile-prefetch-1732304716843 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T19:45:20,135 DEBUG [hfile-prefetch-1732304716843 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49. Will sleep 100ms and try the verification again. 2024-11-22T19:45:20,151 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T19:45:20,151 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T19:45:20,151 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T19:45:20,151 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T19:45:20,158 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T19:45:20,158 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732304720151 2024-11-22T19:45:20,158 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T19:45:20,187 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;982bb8edae1e:35577,5,FailOnTimeoutGroup] 2024-11-22T19:45:20,187 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '982bb8edae1e,35577,1732304717440' ***** 2024-11-22T19:45:20,187 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-22T19:45:20,188 INFO [RS:0;982bb8edae1e:35577 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T19:45:20,188 INFO [RS:0;982bb8edae1e:35577 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.persistence 2024-11-22T19:45:20,188 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T19:45:20,188 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T19:45:20,188 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T19:45:20,188 INFO [RS:0;982bb8edae1e:35577 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T19:45:20,188 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T19:45:20,198 DEBUG [RS:0;982bb8edae1e:35577 {}] bucket.BucketCache(1487): Thread RS:0;982bb8edae1e:35577 started persisting bucket cache to file 2024-11-22T19:45:20,198 DEBUG [RS:0;982bb8edae1e:35577 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T19:45:20,198 DEBUG [RS:0;982bb8edae1e:35577 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T19:45:20,202 DEBUG [RS:0;982bb8edae1e:35577 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T19:45:20,202 DEBUG [RS:0;982bb8edae1e:35577 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732304720198 2024-11-22T19:45:20,202 DEBUG [RS:0;982bb8edae1e:35577 {}] bucket.BucketCache(1509): Thread RS:0;982bb8edae1e:35577 finished persisting bucket cache to file, renaming 2024-11-22T19:45:20,202 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T19:45:20,202 INFO [RS:0;982bb8edae1e:35577 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T19:45:20,202 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T19:45:20,202 INFO [RS:0;982bb8edae1e:35577 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T19:45:20,202 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(3091): Received CLOSE for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,202 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(959): stopping server 982bb8edae1e,35577,1732304717440 2024-11-22T19:45:20,203 INFO [RS:0;982bb8edae1e:35577 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T19:45:20,203 INFO [RS:0;982bb8edae1e:35577 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;982bb8edae1e:35577. 2024-11-22T19:45:20,203 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T19:45:20,203 DEBUG [RS:0;982bb8edae1e:35577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T19:45:20,203 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 83f9081ac3d35c2353cfcd3ee4517925, disabling compactions & flushes 2024-11-22T19:45:20,203 DEBUG [RS:0;982bb8edae1e:35577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:20,203 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:20,203 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:20,203 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T19:45:20,203 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. after waiting 0 ms 2024-11-22T19:45:20,203 DEBUG [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(1325): Online Regions={83f9081ac3d35c2353cfcd3ee4517925=testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925.} 2024-11-22T19:45:20,203 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:20,203 DEBUG [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(1351): Waiting on 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,204 DEBUG [StoreFileCloser-83f9081ac3d35c2353cfcd3ee4517925-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49 2024-11-22T19:45:20,205 WARN [hfile-prefetch-1732304716843 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-22T19:45:20,206 DEBUG [hfile-prefetch-1732304716843 {}] hfile.PrefetchExecutor(121): Prefetch completed for 00fc03c51b1f4448beb9d5b6b8881b49 2024-11-22T19:45:20,208 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T19:45:20,209 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:20,209 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 83f9081ac3d35c2353cfcd3ee4517925: Waiting for close lock at 1732304720203Running coprocessor pre-close hooks at 1732304720203Disabling compacts and flushes for region at 1732304720203Disabling writes for close at 1732304720203Writing region close event to WAL at 1732304720204 (+1 ms)Running coprocessor post-close hooks at 1732304720208 (+4 ms)Closed at 1732304720209 (+1 ms) 2024-11-22T19:45:20,209 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:20,241 INFO [regionserver/982bb8edae1e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:20,403 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(976): stopping server 982bb8edae1e,35577,1732304717440; all regions closed. 2024-11-22T19:45:20,405 DEBUG [RS:0;982bb8edae1e:35577 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs 2024-11-22T19:45:20,405 INFO [RS:0;982bb8edae1e:35577 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 982bb8edae1e%2C35577%2C1732304717440:(num 1732304717978) 2024-11-22T19:45:20,405 DEBUG [RS:0;982bb8edae1e:35577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:20,405 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:20,405 INFO [RS:0;982bb8edae1e:35577 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T19:45:20,406 INFO [RS:0;982bb8edae1e:35577 {}] hbase.ChoreService(370): Chore service for: regionserver/982bb8edae1e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T19:45:20,406 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T19:45:20,406 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T19:45:20,406 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T19:45:20,406 INFO [RS:0;982bb8edae1e:35577 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T19:45:20,406 INFO [regionserver/982bb8edae1e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T19:45:20,406 INFO [RS:0;982bb8edae1e:35577 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35577 2024-11-22T19:45:20,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/982bb8edae1e,35577,1732304717440 2024-11-22T19:45:20,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:20,415 INFO [RS:0;982bb8edae1e:35577 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T19:45:20,417 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [982bb8edae1e,35577,1732304717440] 2024-11-22T19:45:20,418 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/982bb8edae1e,35577,1732304717440 already deleted, retry=false 2024-11-22T19:45:20,418 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 982bb8edae1e,35577,1732304717440 on 982bb8edae1e,36577,1732304717394 2024-11-22T19:45:20,423 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 982bb8edae1e,35577,1732304717440, splitWal=true, meta=false 2024-11-22T19:45:20,425 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 982bb8edae1e,35577,1732304717440 (carryingMeta=false) 982bb8edae1e,35577,1732304717440/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@39980502[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-22T19:45:20,425 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 982bb8edae1e,35577,1732304717440, splitWal=true, meta=false 2024-11-22T19:45:20,426 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 982bb8edae1e,35577,1732304717440 had 1 regions 2024-11-22T19:45:20,427 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 982bb8edae1e,35577,1732304717440, splitWal=true, meta=false, isMeta: false 2024-11-22T19:45:20,427 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35577,1732304717440-splitting 2024-11-22T19:45:20,428 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35577,1732304717440-splitting dir is empty, no logs to split. 2024-11-22T19:45:20,428 INFO [PEWorker-2 {}] master.SplitWALManager(105): 982bb8edae1e,35577,1732304717440 WAL count=0, meta=false 2024-11-22T19:45:20,429 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35577,1732304717440-splitting dir is empty, no logs to split. 2024-11-22T19:45:20,429 INFO [PEWorker-2 {}] master.SplitWALManager(105): 982bb8edae1e,35577,1732304717440 WAL count=0, meta=false 2024-11-22T19:45:20,429 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 982bb8edae1e,35577,1732304717440 WAL splitting is done? wals=0, meta=false 2024-11-22T19:45:20,430 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 982bb8edae1e,35577,1732304717440 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,35577,1732304717440-splitting does not exist 2024-11-22T19:45:20,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN}] 2024-11-22T19:45:20,432 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN 2024-11-22T19:45:20,433 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T19:45:20,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:20,517 INFO [RS:0;982bb8edae1e:35577 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T19:45:20,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35577-0x101413630d20001, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:20,517 INFO [RS:0;982bb8edae1e:35577 {}] regionserver.HRegionServer(1031): Exiting; stopping=982bb8edae1e,35577,1732304717440; zookeeper connection closed. 2024-11-22T19:45:20,518 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@13c08556 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@13c08556 2024-11-22T19:45:20,584 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=83f9081ac3d35c2353cfcd3ee4517925, regionState=OPENING, regionLocation=982bb8edae1e,35919,1732304717474 2024-11-22T19:45:20,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN because future has completed 2024-11-22T19:45:20,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83f9081ac3d35c2353cfcd3ee4517925, server=982bb8edae1e,35919,1732304717474}] 2024-11-22T19:45:20,704 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/982bb8edae1e:0 server-side Connection retries=45 2024-11-22T19:45:20,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:20,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:20,704 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T19:45:20,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T19:45:20,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T19:45:20,704 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T19:45:20,705 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T19:45:20,705 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41555 2024-11-22T19:45:20,707 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41555 connecting to ZooKeeper ensemble=127.0.0.1:53451 2024-11-22T19:45:20,707 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:20,707 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:20,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415550x0, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T19:45:20,711 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:415550x0, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-22T19:45:20,711 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-22T19:45:20,711 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41555-0x101413630d20003 connected 2024-11-22T19:45:20,711 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T19:45:20,715 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.cache 2024-11-22T19:45:20,715 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T19:45:20,715 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T19:45:20,715 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T19:45:20,715 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T19:45:20,716 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T19:45:20,716 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-22T19:45:20,718 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-22T19:45:20,720 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T19:45:20,721 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-22T19:45:20,721 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41555-0x101413630d20003, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T19:45:20,721 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T19:45:20,721 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-22T19:45:20,722 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.persistence is complete. 2024-11-22T19:45:20,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41555-0x101413630d20003, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T19:45:20,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41555 2024-11-22T19:45:20,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41555 2024-11-22T19:45:20,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41555 2024-11-22T19:45:20,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41555 2024-11-22T19:45:20,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41555 2024-11-22T19:45:20,734 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(746): ClusterId : 84354ac9-86be-4d4f-8ec5-886fc7879516 2024-11-22T19:45:20,734 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T19:45:20,737 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T19:45:20,737 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T19:45:20,739 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T19:45:20,739 DEBUG [RS:2;982bb8edae1e:41555 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150e5d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=982bb8edae1e/172.17.0.2:0 2024-11-22T19:45:20,742 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:20,742 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 83f9081ac3d35c2353cfcd3ee4517925, NAME => 'testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925.', STARTKEY => '', ENDKEY => ''} 2024-11-22T19:45:20,743 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732304718506 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,743 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T19:45:20,743 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,743 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,746 INFO [StoreOpener-83f9081ac3d35c2353cfcd3ee4517925-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,747 INFO [StoreOpener-83f9081ac3d35c2353cfcd3ee4517925-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83f9081ac3d35c2353cfcd3ee4517925 columnFamilyName family 2024-11-22T19:45:20,747 DEBUG [StoreOpener-83f9081ac3d35c2353cfcd3ee4517925-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T19:45:20,749 DEBUG [StoreFileOpener-83f9081ac3d35c2353cfcd3ee4517925-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49, delay=986 ms 2024-11-22T19:45:20,751 DEBUG [StoreOpener-83f9081ac3d35c2353cfcd3ee4517925-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49 2024-11-22T19:45:20,751 INFO [StoreOpener-83f9081ac3d35c2353cfcd3ee4517925-1 {}] regionserver.HStore(327): Store=83f9081ac3d35c2353cfcd3ee4517925/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T19:45:20,751 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,752 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,752 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,752 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,752 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,753 DEBUG [RS:2;982bb8edae1e:41555 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;982bb8edae1e:41555 2024-11-22T19:45:20,753 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T19:45:20,753 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T19:45:20,753 DEBUG [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T19:45:20,754 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,754 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(2659): reportForDuty to master=982bb8edae1e,36577,1732304717394 with port=41555, startcode=1732304720704 2024-11-22T19:45:20,754 DEBUG [RS:2;982bb8edae1e:41555 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T19:45:20,754 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 83f9081ac3d35c2353cfcd3ee4517925; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61013150, jitterRate=-0.09083321690559387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T19:45:20,754 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:20,755 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 83f9081ac3d35c2353cfcd3ee4517925: Running coprocessor pre-open hook at 1732304720743Writing region info on filesystem at 1732304720743Initializing all the Stores at 1732304720744 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732304720744Cleaning up temporary data from old regions at 1732304720752 (+8 ms)Running coprocessor post-open hooks at 1732304720754 (+2 ms)Region opened successfully at 1732304720755 (+1 ms) 2024-11-22T19:45:20,756 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925., pid=9, masterSystemTime=1732304720739 2024-11-22T19:45:20,757 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41083, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T19:45:20,758 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 982bb8edae1e,41555,1732304720704 2024-11-22T19:45:20,758 DEBUG [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:20,759 INFO [RS_OPEN_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:20,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36577 {}] master.ServerManager(517): Registering regionserver=982bb8edae1e,41555,1732304720704 2024-11-22T19:45:20,759 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=83f9081ac3d35c2353cfcd3ee4517925, regionState=OPEN, openSeqNum=10, regionLocation=982bb8edae1e,35919,1732304717474 2024-11-22T19:45:20,760 DEBUG [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816 2024-11-22T19:45:20,760 DEBUG [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T19:45:20,760 DEBUG [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T19:45:20,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83f9081ac3d35c2353cfcd3ee4517925, server=982bb8edae1e,35919,1732304717474 because future has completed 2024-11-22T19:45:20,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:20,762 DEBUG [RS:2;982bb8edae1e:41555 {}] zookeeper.ZKUtil(111): regionserver:41555-0x101413630d20003, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/982bb8edae1e,41555,1732304720704 2024-11-22T19:45:20,763 WARN [RS:2;982bb8edae1e:41555 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T19:45:20,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [982bb8edae1e,41555,1732304720704] 2024-11-22T19:45:20,763 INFO [RS:2;982bb8edae1e:41555 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T19:45:20,763 DEBUG [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,41555,1732304720704 2024-11-22T19:45:20,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-22T19:45:20,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 83f9081ac3d35c2353cfcd3ee4517925, server=982bb8edae1e,35919,1732304717474 in 176 msec 2024-11-22T19:45:20,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T19:45:20,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732304718506, region=83f9081ac3d35c2353cfcd3ee4517925, ASSIGN in 334 msec 2024-11-22T19:45:20,766 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 982bb8edae1e,35577,1732304717440 after splitting done 2024-11-22T19:45:20,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 982bb8edae1e,35577,1732304717440, splitWal=true, meta=false in 348 msec 2024-11-22T19:45:20,770 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T19:45:20,776 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T19:45:20,779 INFO [RS:2;982bb8edae1e:41555 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T19:45:20,779 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,780 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T19:45:20,780 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T19:45:20,780 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,780 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/982bb8edae1e:0, corePoolSize=2, maxPoolSize=2 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/982bb8edae1e:0, corePoolSize=1, maxPoolSize=1 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:20,781 DEBUG [RS:2;982bb8edae1e:41555 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/982bb8edae1e:0, corePoolSize=3, maxPoolSize=3 2024-11-22T19:45:20,782 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,782 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,782 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,782 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,782 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,782 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,41555,1732304720704-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T19:45:20,806 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T19:45:20,807 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=982bb8edae1e,41555,1732304720704-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,807 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,807 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.Replication(171): 982bb8edae1e,41555,1732304720704 started 2024-11-22T19:45:20,823 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T19:45:20,823 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(1482): Serving as 982bb8edae1e,41555,1732304720704, RpcServer on 982bb8edae1e/172.17.0.2:41555, sessionid=0x101413630d20003 2024-11-22T19:45:20,823 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T19:45:20,823 DEBUG [RS:2;982bb8edae1e:41555 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 982bb8edae1e,41555,1732304720704 2024-11-22T19:45:20,823 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,41555,1732304720704' 2024-11-22T19:45:20,823 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T19:45:20,824 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T19:45:20,825 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T19:45:20,825 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T19:45:20,825 DEBUG [RS:2;982bb8edae1e:41555 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 982bb8edae1e,41555,1732304720704 2024-11-22T19:45:20,825 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '982bb8edae1e,41555,1732304720704' 2024-11-22T19:45:20,825 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T19:45:20,825 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T19:45:20,826 DEBUG [RS:2;982bb8edae1e:41555 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T19:45:20,826 INFO [RS:2;982bb8edae1e:41555 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T19:45:20,826 INFO [RS:2;982bb8edae1e:41555 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T19:45:20,927 INFO [RS:2;982bb8edae1e:41555 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T19:45:20,927 INFO [RS:2;982bb8edae1e:41555 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=982bb8edae1e%2C41555%2C1732304720704, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,41555,1732304720704, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs, maxLogs=55 2024-11-22T19:45:20,947 INFO [RS:2;982bb8edae1e:41555 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/WALs/982bb8edae1e,41555,1732304720704/982bb8edae1e%2C41555%2C1732304720704.1732304720928 2024-11-22T19:45:20,947 DEBUG [RS:2;982bb8edae1e:41555 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T19:45:21,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T19:45:21,324 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T19:45:21,324 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T19:45:21,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:21,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:21,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T19:45:21,324 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T19:45:21,324 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1061076084, stopped=false 2024-11-22T19:45:21,324 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=982bb8edae1e,36577,1732304717394 2024-11-22T19:45:21,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41555-0x101413630d20003, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:21,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:21,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T19:45:21,326 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T19:45:21,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:21,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:21,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41555-0x101413630d20003, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:21,326 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T19:45:21,327 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T19:45:21,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:21,327 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '982bb8edae1e,35919,1732304717474' ***** 2024-11-22T19:45:21,327 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T19:45:21,327 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:21,327 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '982bb8edae1e,41555,1732304720704' ***** 2024-11-22T19:45:21,327 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41555-0x101413630d20003, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:21,327 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T19:45:21,327 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T19:45:21,327 INFO [RS:1;982bb8edae1e:35919 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T19:45:21,328 INFO [RS:2;982bb8edae1e:41555 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T19:45:21,328 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T19:45:21,328 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T19:45:21,328 INFO [RS:1;982bb8edae1e:35919 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.persistence 2024-11-22T19:45:21,328 INFO [RS:2;982bb8edae1e:41555 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5320d9af-e887-8115-dea2-732b4f5bc885/bucket.persistence 2024-11-22T19:45:21,328 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T19:45:21,328 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T19:45:21,328 INFO [RS:1;982bb8edae1e:35919 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T19:45:21,328 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T19:45:21,328 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T19:45:21,328 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T19:45:21,328 INFO [RS:2;982bb8edae1e:41555 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T19:45:21,328 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T19:45:21,338 DEBUG [RS:1;982bb8edae1e:35919 {}] bucket.BucketCache(1487): Thread RS:1;982bb8edae1e:35919 started persisting bucket cache to file 2024-11-22T19:45:21,338 DEBUG [RS:1;982bb8edae1e:35919 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T19:45:21,339 DEBUG [RS:1;982bb8edae1e:35919 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T19:45:21,339 DEBUG [RS:2;982bb8edae1e:41555 {}] bucket.BucketCache(1487): Thread RS:2;982bb8edae1e:41555 started persisting bucket cache to file 2024-11-22T19:45:21,339 DEBUG [RS:2;982bb8edae1e:41555 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T19:45:21,339 DEBUG [RS:2;982bb8edae1e:41555 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T19:45:21,342 DEBUG [RS:1;982bb8edae1e:35919 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T19:45:21,342 DEBUG [RS:1;982bb8edae1e:35919 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732304721338 2024-11-22T19:45:21,342 DEBUG [RS:2;982bb8edae1e:41555 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T19:45:21,342 DEBUG [RS:2;982bb8edae1e:41555 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732304721339 2024-11-22T19:45:21,342 DEBUG [RS:1;982bb8edae1e:35919 {}] bucket.BucketCache(1509): Thread RS:1;982bb8edae1e:35919 finished persisting bucket cache to file, renaming 2024-11-22T19:45:21,342 DEBUG [RS:2;982bb8edae1e:41555 {}] bucket.BucketCache(1509): Thread RS:2;982bb8edae1e:41555 finished persisting bucket cache to file, renaming 2024-11-22T19:45:21,343 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T19:45:21,343 INFO [RS:1;982bb8edae1e:35919 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T19:45:21,343 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T19:45:21,343 INFO [RS:1;982bb8edae1e:35919 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T19:45:21,343 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(3091): Received CLOSE for 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:21,343 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(959): stopping server 982bb8edae1e,35919,1732304717474 2024-11-22T19:45:21,343 INFO [RS:1;982bb8edae1e:35919 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T19:45:21,343 INFO [RS:1;982bb8edae1e:35919 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;982bb8edae1e:35919. 2024-11-22T19:45:21,343 DEBUG [RS:1;982bb8edae1e:35919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T19:45:21,343 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T19:45:21,343 DEBUG [RS:1;982bb8edae1e:35919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:21,343 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 83f9081ac3d35c2353cfcd3ee4517925, disabling compactions & flushes 2024-11-22T19:45:21,343 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T19:45:21,343 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:21,343 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T19:45:21,344 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T19:45:21,344 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:21,344 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T19:45:21,344 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. after waiting 0 ms 2024-11-22T19:45:21,344 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:21,344 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-22T19:45:21,344 DEBUG [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(1325): Online Regions={83f9081ac3d35c2353cfcd3ee4517925=testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925., 1588230740=hbase:meta,,1.1588230740} 2024-11-22T19:45:21,344 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T19:45:21,344 DEBUG [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 83f9081ac3d35c2353cfcd3ee4517925 2024-11-22T19:45:21,344 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T19:45:21,344 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T19:45:21,344 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T19:45:21,344 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T19:45:21,344 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T19:45:21,344 DEBUG [StoreFileCloser-83f9081ac3d35c2353cfcd3ee4517925-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/family/00fc03c51b1f4448beb9d5b6b8881b49 2024-11-22T19:45:21,344 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-22T19:45:21,345 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T19:45:21,345 INFO [RS:2;982bb8edae1e:41555 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T19:45:21,345 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T19:45:21,345 INFO [RS:2;982bb8edae1e:41555 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T19:45:21,345 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(959): stopping server 982bb8edae1e,41555,1732304720704 2024-11-22T19:45:21,345 INFO [RS:2;982bb8edae1e:41555 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T19:45:21,345 INFO [RS:2;982bb8edae1e:41555 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;982bb8edae1e:41555. 2024-11-22T19:45:21,345 DEBUG [RS:2;982bb8edae1e:41555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T19:45:21,345 DEBUG [RS:2;982bb8edae1e:41555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:21,345 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(976): stopping server 982bb8edae1e,41555,1732304720704; all regions closed. 2024-11-22T19:45:21,347 DEBUG [RS:2;982bb8edae1e:41555 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs 2024-11-22T19:45:21,347 INFO [RS:2;982bb8edae1e:41555 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 982bb8edae1e%2C41555%2C1732304720704:(num 1732304720928) 2024-11-22T19:45:21,347 DEBUG [RS:2;982bb8edae1e:41555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:21,347 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:21,347 INFO [RS:2;982bb8edae1e:41555 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T19:45:21,347 INFO [RS:2;982bb8edae1e:41555 {}] hbase.ChoreService(370): Chore service for: regionserver/982bb8edae1e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T19:45:21,347 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T19:45:21,347 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T19:45:21,348 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T19:45:21,348 INFO [regionserver/982bb8edae1e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T19:45:21,348 INFO [RS:2;982bb8edae1e:41555 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T19:45:21,348 INFO [RS:2;982bb8edae1e:41555 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41555 2024-11-22T19:45:21,350 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/default/testBlockEvictionOnGracefulStop1732304718506/83f9081ac3d35c2353cfcd3ee4517925/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T19:45:21,351 INFO [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:21,351 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 83f9081ac3d35c2353cfcd3ee4517925: Waiting for close lock at 1732304721343Running coprocessor pre-close hooks at 1732304721343Disabling compacts and flushes for region at 1732304721343Disabling writes for close at 1732304721344 (+1 ms)Writing region close event to WAL at 1732304721345 (+1 ms)Running coprocessor post-close hooks at 1732304721351 (+6 ms)Closed at 1732304721351 2024-11-22T19:45:21,351 DEBUG [RS_CLOSE_REGION-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925. 2024-11-22T19:45:21,352 INFO [regionserver/982bb8edae1e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:21,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41555-0x101413630d20003, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/982bb8edae1e,41555,1732304720704 2024-11-22T19:45:21,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:21,356 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 33fcfc72337f49f7bb04fd05dd58444f 2024-11-22T19:45:21,356 INFO [RS:2;982bb8edae1e:41555 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T19:45:21,356 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [982bb8edae1e,41555,1732304720704] 2024-11-22T19:45:21,358 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/982bb8edae1e,41555,1732304720704 already deleted, retry=false 2024-11-22T19:45:21,358 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 982bb8edae1e,41555,1732304720704 expired; onlineServers=1 2024-11-22T19:45:21,369 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/info/33fcfc72337f49f7bb04fd05dd58444f is 209, key is testBlockEvictionOnGracefulStop1732304718506,,1732304718510.83f9081ac3d35c2353cfcd3ee4517925./info:regioninfo/1732304720759/Put/seqid=0 2024-11-22T19:45:21,370 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/info/33fcfc72337f49f7bb04fd05dd58444f 2024-11-22T19:45:21,379 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4dd1c01658e74cb0b7384aeb0f9ff1b8 2024-11-22T19:45:21,383 INFO [regionserver/982bb8edae1e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:21,392 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/ns/4dd1c01658e74cb0b7384aeb0f9ff1b8 is 43, key is default/ns:d/1732304718392/Put/seqid=0 2024-11-22T19:45:21,392 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/ns/4dd1c01658e74cb0b7384aeb0f9ff1b8 2024-11-22T19:45:21,401 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c856fff8b6114c7b9c3caa9de4f51b17 2024-11-22T19:45:21,415 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/table/c856fff8b6114c7b9c3caa9de4f51b17 is 80, key is testBlockEvictionOnGracefulStop1732304718506/table:state/1732304718880/Put/seqid=0 2024-11-22T19:45:21,416 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/table/c856fff8b6114c7b9c3caa9de4f51b17 2024-11-22T19:45:21,418 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/info/33fcfc72337f49f7bb04fd05dd58444f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/info/33fcfc72337f49f7bb04fd05dd58444f 2024-11-22T19:45:21,419 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/info/33fcfc72337f49f7bb04fd05dd58444f, delay=974 ms 2024-11-22T19:45:21,420 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/info/33fcfc72337f49f7bb04fd05dd58444f, entries=14, sequenceid=13, filesize=7.5 K 2024-11-22T19:45:21,421 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/ns/4dd1c01658e74cb0b7384aeb0f9ff1b8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/ns/4dd1c01658e74cb0b7384aeb0f9ff1b8 2024-11-22T19:45:21,422 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/ns/4dd1c01658e74cb0b7384aeb0f9ff1b8, delay=999 ms 2024-11-22T19:45:21,423 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/ns/4dd1c01658e74cb0b7384aeb0f9ff1b8, entries=2, sequenceid=13, filesize=5.0 K 2024-11-22T19:45:21,423 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/.tmp/table/c856fff8b6114c7b9c3caa9de4f51b17 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/table/c856fff8b6114c7b9c3caa9de4f51b17 2024-11-22T19:45:21,424 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/table/c856fff8b6114c7b9c3caa9de4f51b17, delay=966 ms 2024-11-22T19:45:21,425 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/table/c856fff8b6114c7b9c3caa9de4f51b17, entries=2, sequenceid=13, filesize=5.3 K 2024-11-22T19:45:21,426 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=13, compaction requested=false 2024-11-22T19:45:21,426 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/info/33fcfc72337f49f7bb04fd05dd58444f 2024-11-22T19:45:21,427 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/ns/4dd1c01658e74cb0b7384aeb0f9ff1b8 2024-11-22T19:45:21,428 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/table/c856fff8b6114c7b9c3caa9de4f51b17 2024-11-22T19:45:21,432 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-22T19:45:21,432 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T19:45:21,433 INFO [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T19:45:21,433 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732304721344Running coprocessor pre-close hooks at 1732304721344Disabling compacts and flushes for region at 1732304721344Disabling writes for close at 1732304721344Obtaining lock to block concurrent updates at 1732304721344Preparing flush snapshotting stores in 1588230740 at 1732304721344Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732304721345 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732304721345Flushing 1588230740/info: creating writer at 1732304721345Flushing 1588230740/info: appending metadata at 1732304721369 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732304721369Flushing 1588230740/ns: creating writer at 1732304721372 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732304721392 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732304721392Flushing 1588230740/table: creating writer at 1732304721394 (+2 ms)Flushing 1588230740/table: appending metadata at 1732304721414 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732304721414Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d839226: reopening flushed file at 1732304721418 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49360001: reopening flushed file at 1732304721420 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11cfdc3f: reopening flushed file at 1732304721423 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=13, compaction requested=false at 1732304721426 (+3 ms)Writing region close event to WAL at 1732304721428 (+2 ms)Running coprocessor post-close hooks at 1732304721432 (+4 ms)Closed at 1732304721432 2024-11-22T19:45:21,433 DEBUG [RS_CLOSE_META-regionserver/982bb8edae1e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T19:45:21,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41555-0x101413630d20003, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:21,457 INFO [RS:2;982bb8edae1e:41555 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T19:45:21,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41555-0x101413630d20003, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:21,457 INFO [RS:2;982bb8edae1e:41555 {}] regionserver.HRegionServer(1031): Exiting; stopping=982bb8edae1e,41555,1732304720704; zookeeper connection closed. 2024-11-22T19:45:21,458 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7ec6852e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7ec6852e 2024-11-22T19:45:21,544 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(976): stopping server 982bb8edae1e,35919,1732304717474; all regions closed. 2024-11-22T19:45:21,546 DEBUG [RS:1;982bb8edae1e:35919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs 2024-11-22T19:45:21,546 INFO [RS:1;982bb8edae1e:35919 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 982bb8edae1e%2C35919%2C1732304717474.meta:.meta(num 1732304718345) 2024-11-22T19:45:21,548 DEBUG [RS:1;982bb8edae1e:35919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/oldWALs 2024-11-22T19:45:21,548 INFO [RS:1;982bb8edae1e:35919 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 982bb8edae1e%2C35919%2C1732304717474:(num 1732304717986) 2024-11-22T19:45:21,548 DEBUG [RS:1;982bb8edae1e:35919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T19:45:21,548 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T19:45:21,548 INFO [RS:1;982bb8edae1e:35919 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T19:45:21,548 INFO [RS:1;982bb8edae1e:35919 {}] hbase.ChoreService(370): Chore service for: regionserver/982bb8edae1e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T19:45:21,548 INFO [RS:1;982bb8edae1e:35919 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T19:45:21,548 INFO [regionserver/982bb8edae1e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T19:45:21,549 INFO [RS:1;982bb8edae1e:35919 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35919 2024-11-22T19:45:21,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/982bb8edae1e,35919,1732304717474 2024-11-22T19:45:21,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T19:45:21,551 INFO [RS:1;982bb8edae1e:35919 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T19:45:21,554 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [982bb8edae1e,35919,1732304717474] 2024-11-22T19:45:21,556 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/982bb8edae1e,35919,1732304717474 already deleted, retry=false 2024-11-22T19:45:21,556 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 982bb8edae1e,35919,1732304717474 expired; onlineServers=0 2024-11-22T19:45:21,556 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '982bb8edae1e,36577,1732304717394' ***** 2024-11-22T19:45:21,556 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T19:45:21,556 INFO [M:0;982bb8edae1e:36577 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T19:45:21,556 INFO [M:0;982bb8edae1e:36577 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T19:45:21,556 DEBUG [M:0;982bb8edae1e:36577 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T19:45:21,556 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T19:45:21,556 DEBUG [M:0;982bb8edae1e:36577 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T19:45:21,556 DEBUG [master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.large.0-1732304717715 {}] cleaner.HFileCleaner(306): Exit Thread[master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.large.0-1732304717715,5,FailOnTimeoutGroup] 2024-11-22T19:45:21,556 DEBUG [master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.small.0-1732304717715 {}] cleaner.HFileCleaner(306): Exit Thread[master/982bb8edae1e:0:becomeActiveMaster-HFileCleaner.small.0-1732304717715,5,FailOnTimeoutGroup] 2024-11-22T19:45:21,557 INFO [M:0;982bb8edae1e:36577 {}] hbase.ChoreService(370): Chore service for: master/982bb8edae1e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T19:45:21,557 INFO [M:0;982bb8edae1e:36577 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T19:45:21,557 DEBUG [M:0;982bb8edae1e:36577 {}] master.HMaster(1795): Stopping service threads 2024-11-22T19:45:21,557 INFO [M:0;982bb8edae1e:36577 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T19:45:21,557 INFO [M:0;982bb8edae1e:36577 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T19:45:21,557 INFO [M:0;982bb8edae1e:36577 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T19:45:21,557 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T19:45:21,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T19:45:21,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T19:45:21,558 DEBUG [M:0;982bb8edae1e:36577 {}] zookeeper.ZKUtil(347): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T19:45:21,558 WARN [M:0;982bb8edae1e:36577 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T19:45:21,558 INFO [M:0;982bb8edae1e:36577 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/.lastflushedseqids 2024-11-22T19:45:21,566 INFO [M:0;982bb8edae1e:36577 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T19:45:21,566 INFO [M:0;982bb8edae1e:36577 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T19:45:21,566 DEBUG [M:0;982bb8edae1e:36577 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T19:45:21,566 INFO [M:0;982bb8edae1e:36577 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:21,566 DEBUG [M:0;982bb8edae1e:36577 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:21,566 DEBUG [M:0;982bb8edae1e:36577 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T19:45:21,566 DEBUG [M:0;982bb8edae1e:36577 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:21,566 INFO [M:0;982bb8edae1e:36577 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-22T19:45:21,576 DEBUG [M:0;982bb8edae1e:36577 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 48451e3a58dc4c34aaa216c98d278fca 2024-11-22T19:45:21,589 DEBUG [M:0;982bb8edae1e:36577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/48451e3a58dc4c34aaa216c98d278fca is 82, key is hbase:meta,,1/info:regioninfo/1732304718379/Put/seqid=0 2024-11-22T19:45:21,590 INFO [M:0;982bb8edae1e:36577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/48451e3a58dc4c34aaa216c98d278fca 2024-11-22T19:45:21,597 DEBUG [M:0;982bb8edae1e:36577 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fbafcaf6779149cd801f3159de28004b 2024-11-22T19:45:21,611 DEBUG [M:0;982bb8edae1e:36577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbafcaf6779149cd801f3159de28004b is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732304718884/Put/seqid=0 2024-11-22T19:45:21,612 INFO [M:0;982bb8edae1e:36577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbafcaf6779149cd801f3159de28004b 2024-11-22T19:45:21,620 DEBUG [M:0;982bb8edae1e:36577 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 58f4c0b6af294977a20adc04b00a6d17 2024-11-22T19:45:21,634 DEBUG [M:0;982bb8edae1e:36577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58f4c0b6af294977a20adc04b00a6d17 is 69, key is 982bb8edae1e,35919,1732304717474/rs:state/1732304717826/Put/seqid=0 2024-11-22T19:45:21,635 INFO [M:0;982bb8edae1e:36577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58f4c0b6af294977a20adc04b00a6d17 2024-11-22T19:45:21,637 INFO [M:0;982bb8edae1e:36577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 58f4c0b6af294977a20adc04b00a6d17 2024-11-22T19:45:21,637 DEBUG [M:0;982bb8edae1e:36577 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/48451e3a58dc4c34aaa216c98d278fca as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/48451e3a58dc4c34aaa216c98d278fca 2024-11-22T19:45:21,639 INFO [M:0;982bb8edae1e:36577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/48451e3a58dc4c34aaa216c98d278fca, entries=8, sequenceid=83, filesize=5.5 K 2024-11-22T19:45:21,640 DEBUG [M:0;982bb8edae1e:36577 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbafcaf6779149cd801f3159de28004b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbafcaf6779149cd801f3159de28004b 2024-11-22T19:45:21,641 INFO [M:0;982bb8edae1e:36577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbafcaf6779149cd801f3159de28004b, entries=9, sequenceid=83, filesize=6.4 K 2024-11-22T19:45:21,641 DEBUG [M:0;982bb8edae1e:36577 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58f4c0b6af294977a20adc04b00a6d17 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/58f4c0b6af294977a20adc04b00a6d17 2024-11-22T19:45:21,643 INFO [M:0;982bb8edae1e:36577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 58f4c0b6af294977a20adc04b00a6d17 2024-11-22T19:45:21,643 INFO [M:0;982bb8edae1e:36577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/45126be5-e690-60f8-5973-93ab699447d4/fdeef7a8-f771-48ae-9492-ffadcde2c816/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/58f4c0b6af294977a20adc04b00a6d17, entries=3, sequenceid=83, filesize=5.3 K 2024-11-22T19:45:21,644 INFO [M:0;982bb8edae1e:36577 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false 2024-11-22T19:45:21,645 INFO [M:0;982bb8edae1e:36577 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T19:45:21,645 DEBUG [M:0;982bb8edae1e:36577 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732304721566Disabling compacts and flushes for region at 1732304721566Disabling writes for close at 1732304721566Obtaining lock to block concurrent updates at 1732304721567 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732304721567Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1732304721567Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732304721567Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732304721567Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732304721589 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732304721589Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732304721591 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732304721611 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732304721611Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732304721614 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732304721634 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732304721634Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c169f36: reopening flushed file at 1732304721637 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@382bc1c0: reopening flushed file at 1732304721639 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61589615: reopening flushed file at 1732304721641 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false at 1732304721644 (+3 ms)Writing region close event to WAL at 1732304721645 (+1 ms)Closed at 1732304721645 2024-11-22T19:45:21,646 INFO [M:0;982bb8edae1e:36577 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T19:45:21,646 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T19:45:21,647 INFO [M:0;982bb8edae1e:36577 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36577 2024-11-22T19:45:21,647 INFO [M:0;982bb8edae1e:36577 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T19:45:21,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:21,654 INFO [RS:1;982bb8edae1e:35919 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T19:45:21,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x101413630d20002, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:21,654 INFO [RS:1;982bb8edae1e:35919 {}] regionserver.HRegionServer(1031): Exiting; stopping=982bb8edae1e,35919,1732304717474; zookeeper connection closed. 2024-11-22T19:45:21,655 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@52fea24c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@52fea24c 2024-11-22T19:45:21,655 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-22T19:45:21,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:21,749 INFO [M:0;982bb8edae1e:36577 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T19:45:21,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36577-0x101413630d20000, quorum=127.0.0.1:53451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T19:45:21,752 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T19:45:21,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T19:45:21,753 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T19:45:21,760 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=288 (was 278) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8637 (was 8745)