2024-11-26 20:45:11,825 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-26 20:45:11,844 main DEBUG Took 0.010333 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 20:45:11,844 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 20:45:11,845 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 20:45:11,845 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 20:45:11,847 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,853 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 20:45:11,865 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,867 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,868 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,868 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,868 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,869 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,869 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,870 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,870 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,871 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,871 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,872 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,872 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,873 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,873 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,874 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,874 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,875 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,875 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,875 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,875 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,876 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,876 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:45:11,877 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,877 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 20:45:11,878 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:45:11,879 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 20:45:11,881 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 20:45:11,882 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 20:45:11,883 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 20:45:11,883 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 20:45:11,892 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 20:45:11,894 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 20:45:11,896 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 20:45:11,896 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 20:45:11,896 main DEBUG createAppenders(={Console}) 2024-11-26 20:45:11,897 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-26 20:45:11,897 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-26 20:45:11,897 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-26 20:45:11,898 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 20:45:11,898 main DEBUG OutputStream closed 2024-11-26 20:45:11,898 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 20:45:11,899 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 20:45:11,899 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-26 20:45:11,967 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 20:45:11,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 20:45:11,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 20:45:11,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 20:45:11,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 20:45:11,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 20:45:11,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 20:45:11,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 20:45:11,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 20:45:11,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 20:45:11,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 20:45:11,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 20:45:11,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 20:45:11,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 20:45:11,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 20:45:11,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 20:45:11,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 20:45:11,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 20:45:11,978 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 20:45:11,979 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-26 20:45:11,979 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 20:45:11,980 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-26T20:45:11,994 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-26 20:45:11,996 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 20:45:11,997 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T20:45:12,258 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0 2024-11-26T20:45:12,283 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=419, ProcessCount=11, AvailableMemoryMB=6451 2024-11-26T20:45:12,373 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T20:45:12,458 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:12,473 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/cluster_2c0512ec-7752-8016-745f-a5e104d4687c, deleteOnExit=true 2024-11-26T20:45:12,561 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/cluster_2c0512ec-7752-8016-745f-a5e104d4687c/zookeeper_0, clientPort=49272, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/cluster_2c0512ec-7752-8016-745f-a5e104d4687c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/cluster_2c0512ec-7752-8016-745f-a5e104d4687c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T20:45:12,573 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49272 2024-11-26T20:45:12,574 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:12,658 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 with version=8 2024-11-26T20:45:12,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/hbase-staging 2024-11-26T20:45:12,755 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T20:45:13,026 INFO [Time-limited test {}] client.ConnectionUtils(128): master/cd6be161543e:0 server-side Connection retries=45 2024-11-26T20:45:13,037 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:13,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:13,046 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:45:13,046 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:13,046 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:45:13,184 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T20:45:13,272 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:45:13,310 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T20:45:13,320 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T20:45:13,324 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:45:13,365 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14630 (auto-detected) 2024-11-26T20:45:13,367 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T20:45:13,392 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45865 2024-11-26T20:45:13,425 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45865 connecting to ZooKeeper ensemble=127.0.0.1:49272 2024-11-26T20:45:13,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:458650x0, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:45:13,465 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45865-0x10156067c6d0000 connected 2024-11-26T20:45:13,493 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:13,494 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:13,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:13,507 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31, hbase.cluster.distributed=false 2024-11-26T20:45:13,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:45:13,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45865 2024-11-26T20:45:13,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45865 2024-11-26T20:45:13,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45865 2024-11-26T20:45:13,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45865 2024-11-26T20:45:13,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45865 2024-11-26T20:45:13,637 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cd6be161543e:0 server-side Connection retries=45 2024-11-26T20:45:13,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:13,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:13,640 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:45:13,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:13,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:45:13,644 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T20:45:13,647 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:45:13,649 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42287 2024-11-26T20:45:13,651 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42287 connecting to ZooKeeper ensemble=127.0.0.1:49272 2024-11-26T20:45:13,652 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:13,653 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:13,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422870x0, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:45:13,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42287-0x10156067c6d0001 connected 2024-11-26T20:45:13,658 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:13,664 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T20:45:13,677 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.cache 2024-11-26T20:45:13,682 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T20:45:13,683 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T20:45:13,685 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T20:45:13,687 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T20:45:13,687 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T20:45:13,692 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T20:45:13,697 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T20:45:13,699 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T20:45:13,700 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.persistence is complete. 2024-11-26T20:45:13,700 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T20:45:13,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:45:13,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42287 2024-11-26T20:45:13,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42287 2024-11-26T20:45:13,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42287 2024-11-26T20:45:13,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42287 2024-11-26T20:45:13,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42287 2024-11-26T20:45:13,731 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cd6be161543e:0 server-side Connection retries=45 2024-11-26T20:45:13,731 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:13,732 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:13,732 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:45:13,732 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:13,732 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:45:13,732 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T20:45:13,733 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:45:13,734 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42887 2024-11-26T20:45:13,736 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42887 connecting to ZooKeeper ensemble=127.0.0.1:49272 2024-11-26T20:45:13,736 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:13,737 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:13,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428870x0, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:45:13,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42887-0x10156067c6d0002 connected 2024-11-26T20:45:13,743 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:13,744 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T20:45:13,745 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.cache 2024-11-26T20:45:13,746 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T20:45:13,746 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T20:45:13,746 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T20:45:13,748 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T20:45:13,748 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T20:45:13,748 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T20:45:13,749 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T20:45:13,749 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.persistence is complete. 2024-11-26T20:45:13,750 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T20:45:13,751 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T20:45:13,753 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:45:13,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42887 2024-11-26T20:45:13,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42887 2024-11-26T20:45:13,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42887 2024-11-26T20:45:13,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42887 2024-11-26T20:45:13,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42887 2024-11-26T20:45:13,777 DEBUG [M:0;cd6be161543e:45865 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cd6be161543e:45865 2024-11-26T20:45:13,778 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cd6be161543e,45865,1732653912827 2024-11-26T20:45:13,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:13,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:13,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:13,789 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cd6be161543e,45865,1732653912827 2024-11-26T20:45:13,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T20:45:13,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T20:45:13,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:13,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:13,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:13,818 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T20:45:13,820 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cd6be161543e,45865,1732653912827 from backup master directory 2024-11-26T20:45:13,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cd6be161543e,45865,1732653912827 2024-11-26T20:45:13,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:13,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:13,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:13,824 WARN [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:45:13,825 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cd6be161543e,45865,1732653912827 2024-11-26T20:45:13,827 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T20:45:13,829 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T20:45:13,836 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/hbase.id] with ID: 3aee14e6-f228-4b08-a122-b3851c1a77cb 2024-11-26T20:45:13,837 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/.tmp/hbase.id 2024-11-26T20:45:13,842 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/hbase.id] 2024-11-26T20:45:13,886 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:13,892 INFO [master/cd6be161543e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T20:45:13,919 INFO [master/cd6be161543e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-26T20:45:13,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:13,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:13,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:13,952 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:45:13,955 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T20:45:14,010 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:45:14,018 INFO [master/cd6be161543e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:45:14,078 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store 2024-11-26T20:45:14,108 INFO [master/cd6be161543e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T20:45:14,112 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:14,113 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T20:45:14,113 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:14,114 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:14,115 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T20:45:14,115 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:14,116 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:14,116 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732653914113Disabling compacts and flushes for region at 1732653914113Disabling writes for close at 1732653914115 (+2 ms)Writing region close event to WAL at 1732653914116 (+1 ms)Closed at 1732653914116 2024-11-26T20:45:14,117 WARN [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/.initializing 2024-11-26T20:45:14,117 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/WALs/cd6be161543e,45865,1732653912827 2024-11-26T20:45:14,129 INFO [master/cd6be161543e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:45:14,142 INFO [master/cd6be161543e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cd6be161543e%2C45865%2C1732653912827, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/WALs/cd6be161543e,45865,1732653912827, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/oldWALs, maxLogs=10 2024-11-26T20:45:14,180 INFO [master/cd6be161543e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/WALs/cd6be161543e,45865,1732653912827/cd6be161543e%2C45865%2C1732653912827.1732653914146 2024-11-26T20:45:14,181 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T20:45:14,181 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:45:14,182 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:14,185 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,186 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,266 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T20:45:14,270 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:14,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:14,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T20:45:14,279 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:14,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:14,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,285 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T20:45:14,285 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:14,285 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:14,286 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,290 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T20:45:14,290 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:14,291 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:14,291 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,295 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,295 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,298 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,299 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,301 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:45:14,303 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:14,312 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:45:14,313 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60009974, jitterRate=-0.10578170418739319}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:45:14,319 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732653914199Initializing all the Stores at 1732653914199Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653914200 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653914201 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653914201Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653914201Cleaning up temporary data from old regions at 1732653914299 (+98 ms)Region opened successfully at 1732653914319 (+20 ms) 2024-11-26T20:45:14,320 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T20:45:14,353 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc653b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cd6be161543e/172.17.0.2:0 2024-11-26T20:45:14,387 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T20:45:14,401 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T20:45:14,401 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T20:45:14,404 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:45:14,405 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:45:14,412 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-26T20:45:14,412 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T20:45:14,439 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T20:45:14,448 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T20:45:14,451 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T20:45:14,453 INFO [master/cd6be161543e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T20:45:14,455 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T20:45:14,457 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T20:45:14,459 INFO [master/cd6be161543e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T20:45:14,462 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T20:45:14,463 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T20:45:14,465 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T20:45:14,468 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T20:45:14,486 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T20:45:14,488 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T20:45:14,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:14,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:14,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:14,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:14,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:14,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:14,504 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=cd6be161543e,45865,1732653912827, sessionid=0x10156067c6d0000, setting cluster-up flag (Was=false) 2024-11-26T20:45:14,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:14,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:14,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:14,526 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T20:45:14,528 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cd6be161543e,45865,1732653912827 2024-11-26T20:45:14,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:14,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:14,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:14,540 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T20:45:14,541 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cd6be161543e,45865,1732653912827 2024-11-26T20:45:14,546 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T20:45:14,564 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(746): ClusterId : 3aee14e6-f228-4b08-a122-b3851c1a77cb 2024-11-26T20:45:14,564 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(746): ClusterId : 3aee14e6-f228-4b08-a122-b3851c1a77cb 2024-11-26T20:45:14,567 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T20:45:14,567 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T20:45:14,572 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T20:45:14,572 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T20:45:14,572 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T20:45:14,572 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T20:45:14,577 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T20:45:14,577 DEBUG [RS:1;cd6be161543e:42887 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ba7b8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cd6be161543e/172.17.0.2:0 2024-11-26T20:45:14,577 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T20:45:14,578 DEBUG [RS:0;cd6be161543e:42287 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bcf38b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cd6be161543e/172.17.0.2:0 2024-11-26T20:45:14,592 DEBUG [RS:1;cd6be161543e:42887 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;cd6be161543e:42887 2024-11-26T20:45:14,595 INFO [RS:1;cd6be161543e:42887 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T20:45:14,595 INFO [RS:1;cd6be161543e:42887 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T20:45:14,595 DEBUG [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T20:45:14,596 DEBUG [RS:0;cd6be161543e:42287 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;cd6be161543e:42287 2024-11-26T20:45:14,596 INFO [RS:0;cd6be161543e:42287 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T20:45:14,596 INFO [RS:0;cd6be161543e:42287 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T20:45:14,596 DEBUG [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T20:45:14,598 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(2659): reportForDuty to master=cd6be161543e,45865,1732653912827 with port=42287, startcode=1732653913595 2024-11-26T20:45:14,598 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(2659): reportForDuty to master=cd6be161543e,45865,1732653912827 with port=42887, startcode=1732653913730 2024-11-26T20:45:14,611 DEBUG [RS:0;cd6be161543e:42287 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T20:45:14,611 DEBUG [RS:1;cd6be161543e:42887 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T20:45:14,625 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:45:14,637 INFO [master/cd6be161543e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T20:45:14,645 INFO [master/cd6be161543e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T20:45:14,650 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: cd6be161543e,45865,1732653912827 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T20:45:14,661 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:45:14,662 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:45:14,662 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:45:14,663 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:45:14,663 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/cd6be161543e:0, corePoolSize=10, maxPoolSize=10 2024-11-26T20:45:14,663 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,663 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/cd6be161543e:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:45:14,664 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,673 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732653944673 2024-11-26T20:45:14,674 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:45:14,675 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T20:45:14,675 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T20:45:14,676 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T20:45:14,679 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T20:45:14,680 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T20:45:14,680 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T20:45:14,680 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T20:45:14,681 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:14,681 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T20:45:14,687 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,693 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T20:45:14,694 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45161, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T20:45:14,694 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59525, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T20:45:14,694 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T20:45:14,695 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T20:45:14,698 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T20:45:14,698 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T20:45:14,700 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.large.0-1732653914699,5,FailOnTimeoutGroup] 2024-11-26T20:45:14,701 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.small.0-1732653914700,5,FailOnTimeoutGroup] 2024-11-26T20:45:14,701 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,701 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T20:45:14,702 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,702 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45865 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:45:14,703 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,704 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T20:45:14,704 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 2024-11-26T20:45:14,710 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45865 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cd6be161543e,42887,1732653913730 2024-11-26T20:45:14,713 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45865 {}] master.ServerManager(517): Registering regionserver=cd6be161543e,42887,1732653913730 2024-11-26T20:45:14,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:14,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T20:45:14,723 DEBUG [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 2024-11-26T20:45:14,723 DEBUG [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T20:45:14,724 DEBUG [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T20:45:14,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:45:14,730 DEBUG [RS:1;cd6be161543e:42887 {}] zookeeper.ZKUtil(111): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cd6be161543e,42887,1732653913730 2024-11-26T20:45:14,730 WARN [RS:1;cd6be161543e:42887 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:45:14,730 INFO [RS:1;cd6be161543e:42887 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:45:14,731 DEBUG [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,42887,1732653913730 2024-11-26T20:45:14,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T20:45:14,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:14,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:14,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T20:45:14,735 DEBUG [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T20:45:14,736 WARN [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T20:45:14,738 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cd6be161543e,42887,1732653913730] 2024-11-26T20:45:14,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T20:45:14,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:14,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:14,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T20:45:14,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T20:45:14,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:14,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:14,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T20:45:14,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T20:45:14,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:14,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:14,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T20:45:14,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740 2024-11-26T20:45:14,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740 2024-11-26T20:45:14,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T20:45:14,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T20:45:14,761 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:45:14,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T20:45:14,765 INFO [RS:1;cd6be161543e:42887 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T20:45:14,769 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:45:14,770 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71363982, jitterRate=0.06340619921684265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:45:14,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732653914714Initializing all the Stores at 1732653914714Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653914714Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653914716 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653914716Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653914716Cleaning up temporary data from old regions at 1732653914760 (+44 ms)Region opened successfully at 1732653914775 (+15 ms) 2024-11-26T20:45:14,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T20:45:14,776 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T20:45:14,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T20:45:14,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T20:45:14,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T20:45:14,777 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T20:45:14,778 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732653914775Disabling compacts and flushes for region at 1732653914775Disabling writes for close at 1732653914776 (+1 ms)Writing region close event to WAL at 1732653914777 (+1 ms)Closed at 1732653914777 2024-11-26T20:45:14,780 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:45:14,781 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T20:45:14,786 INFO [RS:1;cd6be161543e:42887 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T20:45:14,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T20:45:14,793 INFO [RS:1;cd6be161543e:42887 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T20:45:14,793 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,794 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T20:45:14,798 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:45:14,800 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T20:45:14,800 INFO [RS:1;cd6be161543e:42887 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T20:45:14,802 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,803 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,803 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,803 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,803 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,804 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,804 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cd6be161543e:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:45:14,804 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,804 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,804 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,805 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,805 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,805 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,806 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:14,806 DEBUG [RS:1;cd6be161543e:42887 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:14,807 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,807 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,807 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,807 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,808 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,808 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42887,1732653913730-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:45:14,828 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T20:45:14,829 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42887,1732653913730-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,830 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,830 INFO [RS:1;cd6be161543e:42887 {}] regionserver.Replication(171): cd6be161543e,42887,1732653913730 started 2024-11-26T20:45:14,837 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(2659): reportForDuty to master=cd6be161543e,45865,1732653912827 with port=42287, startcode=1732653913595 2024-11-26T20:45:14,839 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45865 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cd6be161543e,42287,1732653913595 2024-11-26T20:45:14,839 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45865 {}] master.ServerManager(517): Registering regionserver=cd6be161543e,42287,1732653913595 2024-11-26T20:45:14,841 DEBUG [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 2024-11-26T20:45:14,841 DEBUG [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T20:45:14,841 DEBUG [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T20:45:14,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:45:14,847 DEBUG [RS:0;cd6be161543e:42287 {}] zookeeper.ZKUtil(111): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cd6be161543e,42287,1732653913595 2024-11-26T20:45:14,847 WARN [RS:0;cd6be161543e:42287 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:45:14,848 INFO [RS:0;cd6be161543e:42287 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:45:14,848 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cd6be161543e,42287,1732653913595] 2024-11-26T20:45:14,848 DEBUG [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,42287,1732653913595 2024-11-26T20:45:14,850 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,850 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(1482): Serving as cd6be161543e,42887,1732653913730, RpcServer on cd6be161543e/172.17.0.2:42887, sessionid=0x10156067c6d0002 2024-11-26T20:45:14,851 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T20:45:14,851 DEBUG [RS:1;cd6be161543e:42887 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cd6be161543e,42887,1732653913730 2024-11-26T20:45:14,851 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,42887,1732653913730' 2024-11-26T20:45:14,852 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T20:45:14,853 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T20:45:14,854 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T20:45:14,854 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T20:45:14,854 DEBUG [RS:1;cd6be161543e:42887 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cd6be161543e,42887,1732653913730 2024-11-26T20:45:14,854 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,42887,1732653913730' 2024-11-26T20:45:14,854 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T20:45:14,854 INFO [RS:0;cd6be161543e:42287 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T20:45:14,855 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T20:45:14,855 DEBUG [RS:1;cd6be161543e:42887 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T20:45:14,855 INFO [RS:1;cd6be161543e:42887 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T20:45:14,856 INFO [RS:1;cd6be161543e:42887 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T20:45:14,858 INFO [RS:0;cd6be161543e:42287 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T20:45:14,860 INFO [RS:0;cd6be161543e:42287 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T20:45:14,861 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,861 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T20:45:14,861 INFO [RS:0;cd6be161543e:42287 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T20:45:14,862 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,862 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,862 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,862 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,862 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,862 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,862 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cd6be161543e:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:45:14,862 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,863 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,863 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,863 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,863 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,863 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:14,863 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:14,863 DEBUG [RS:0;cd6be161543e:42287 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:14,864 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,864 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,864 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,864 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,864 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,864 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42287,1732653913595-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:45:14,881 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T20:45:14,882 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42287,1732653913595-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,882 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,882 INFO [RS:0;cd6be161543e:42287 {}] regionserver.Replication(171): cd6be161543e,42287,1732653913595 started 2024-11-26T20:45:14,899 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:14,900 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(1482): Serving as cd6be161543e,42287,1732653913595, RpcServer on cd6be161543e/172.17.0.2:42287, sessionid=0x10156067c6d0001 2024-11-26T20:45:14,900 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T20:45:14,900 DEBUG [RS:0;cd6be161543e:42287 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cd6be161543e,42287,1732653913595 2024-11-26T20:45:14,900 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,42287,1732653913595' 2024-11-26T20:45:14,900 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T20:45:14,901 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T20:45:14,902 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T20:45:14,902 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T20:45:14,902 DEBUG [RS:0;cd6be161543e:42287 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cd6be161543e,42287,1732653913595 2024-11-26T20:45:14,902 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,42287,1732653913595' 2024-11-26T20:45:14,902 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T20:45:14,903 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T20:45:14,904 DEBUG [RS:0;cd6be161543e:42287 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T20:45:14,904 INFO [RS:0;cd6be161543e:42287 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T20:45:14,904 INFO [RS:0;cd6be161543e:42287 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T20:45:14,951 WARN [cd6be161543e:45865 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T20:45:14,961 INFO [RS:1;cd6be161543e:42887 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:45:14,962 INFO [RS:1;cd6be161543e:42887 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=cd6be161543e%2C42887%2C1732653913730, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,42887,1732653913730, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs, maxLogs=55 2024-11-26T20:45:14,983 INFO [RS:1;cd6be161543e:42887 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,42887,1732653913730/cd6be161543e%2C42887%2C1732653913730.1732653914964 2024-11-26T20:45:14,985 DEBUG [RS:1;cd6be161543e:42887 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T20:45:15,005 INFO [RS:0;cd6be161543e:42287 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:45:15,006 INFO [RS:0;cd6be161543e:42287 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=cd6be161543e%2C42287%2C1732653913595, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,42287,1732653913595, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs, maxLogs=55 2024-11-26T20:45:15,027 INFO [RS:0;cd6be161543e:42287 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,42287,1732653913595/cd6be161543e%2C42287%2C1732653913595.1732653915008 2024-11-26T20:45:15,027 DEBUG [RS:0;cd6be161543e:42287 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T20:45:15,204 DEBUG [cd6be161543e:45865 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T20:45:15,216 DEBUG [cd6be161543e:45865 {}] balancer.BalancerClusterState(204): Hosts are {cd6be161543e=0} racks are {/default-rack=0} 2024-11-26T20:45:15,219 DEBUG [cd6be161543e:45865 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T20:45:15,219 DEBUG [cd6be161543e:45865 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T20:45:15,219 DEBUG [cd6be161543e:45865 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T20:45:15,219 DEBUG [cd6be161543e:45865 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T20:45:15,219 INFO [cd6be161543e:45865 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T20:45:15,219 INFO [cd6be161543e:45865 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T20:45:15,219 DEBUG [cd6be161543e:45865 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T20:45:15,227 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=cd6be161543e,42887,1732653913730 2024-11-26T20:45:15,232 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cd6be161543e,42887,1732653913730, state=OPENING 2024-11-26T20:45:15,238 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T20:45:15,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:15,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:15,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:15,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:15,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:15,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:15,243 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:45:15,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=cd6be161543e,42887,1732653913730}] 2024-11-26T20:45:15,418 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T20:45:15,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41447, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T20:45:15,429 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T20:45:15,429 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:45:15,430 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T20:45:15,432 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=cd6be161543e%2C42887%2C1732653913730.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,42887,1732653913730, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs, maxLogs=55 2024-11-26T20:45:15,454 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,42887,1732653913730/cd6be161543e%2C42887%2C1732653913730.meta.1732653915433.meta 2024-11-26T20:45:15,455 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T20:45:15,456 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:45:15,458 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T20:45:15,460 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T20:45:15,465 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T20:45:15,469 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T20:45:15,470 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:15,470 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T20:45:15,471 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T20:45:15,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T20:45:15,473 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T20:45:15,473 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:15,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:15,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T20:45:15,474 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T20:45:15,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:15,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:15,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T20:45:15,475 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T20:45:15,476 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:15,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:15,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T20:45:15,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T20:45:15,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:15,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:15,477 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T20:45:15,478 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740 2024-11-26T20:45:15,479 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740 2024-11-26T20:45:15,479 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T20:45:15,480 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T20:45:15,480 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:45:15,482 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T20:45:15,483 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66506580, jitterRate=-0.008974730968475342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:45:15,483 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T20:45:15,485 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732653915471Writing region info on filesystem at 1732653915471Initializing all the Stores at 1732653915472 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653915472Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653915472Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653915472Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653915472Cleaning up temporary data from old regions at 1732653915480 (+8 ms)Running coprocessor post-open hooks at 1732653915483 (+3 ms)Region opened successfully at 1732653915485 (+2 ms) 2024-11-26T20:45:15,492 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732653915409 2024-11-26T20:45:15,502 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T20:45:15,503 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T20:45:15,503 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,42887,1732653913730 2024-11-26T20:45:15,504 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cd6be161543e,42887,1732653913730, state=OPEN 2024-11-26T20:45:15,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:45:15,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:45:15,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:45:15,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:15,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:15,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:15,507 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=cd6be161543e,42887,1732653913730 2024-11-26T20:45:15,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T20:45:15,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=cd6be161543e,42887,1732653913730 in 264 msec 2024-11-26T20:45:15,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T20:45:15,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 723 msec 2024-11-26T20:45:15,515 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:45:15,515 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T20:45:15,542 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:45:15,544 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,42887,1732653913730, seqNum=-1] 2024-11-26T20:45:15,565 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:45:15,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46351, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:45:15,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0070 sec 2024-11-26T20:45:15,583 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732653915582, completionTime=-1 2024-11-26T20:45:15,585 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-26T20:45:15,585 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:45:15,610 INFO [master/cd6be161543e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:45:15,610 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732653975610 2024-11-26T20:45:15,610 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732654035610 2024-11-26T20:45:15,611 INFO [master/cd6be161543e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-26T20:45:15,613 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:45:15,619 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,45865,1732653912827-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:15,619 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,45865,1732653912827-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:15,619 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,45865,1732653912827-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:15,620 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-cd6be161543e:45865, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:15,621 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:15,621 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:15,633 DEBUG [master/cd6be161543e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T20:45:15,654 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.829sec 2024-11-26T20:45:15,656 INFO [master/cd6be161543e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T20:45:15,657 INFO [master/cd6be161543e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T20:45:15,658 INFO [master/cd6be161543e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T20:45:15,659 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T20:45:15,659 INFO [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T20:45:15,660 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,45865,1732653912827-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:45:15,661 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,45865,1732653912827-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T20:45:15,662 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T20:45:15,662 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T20:45:15,663 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,45865,1732653912827-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:15,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3379c249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:45:15,676 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T20:45:15,676 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T20:45:15,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,45865,-1 for getting cluster id 2024-11-26T20:45:15,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:45:15,691 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3aee14e6-f228-4b08-a122-b3851c1a77cb' 2024-11-26T20:45:15,693 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:45:15,693 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3aee14e6-f228-4b08-a122-b3851c1a77cb" 2024-11-26T20:45:15,694 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30d006a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:45:15,694 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,45865,-1] 2024-11-26T20:45:15,696 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:45:15,697 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:15,699 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:45:15,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f573c49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:45:15,702 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:45:15,708 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,42887,1732653913730, seqNum=-1] 2024-11-26T20:45:15,709 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:45:15,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:45:15,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=cd6be161543e,45865,1732653912827 2024-11-26T20:45:15,748 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:45:15,753 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is cd6be161543e,45865,1732653912827 2024-11-26T20:45:15,756 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4dd06b03 2024-11-26T20:45:15,758 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:45:15,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:45:15,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732653915731', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:45:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732653915731 2024-11-26T20:45:15,777 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732653915731 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T20:45:15,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732653915731" procId is: 4 2024-11-26T20:45:15,779 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:15,780 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732653915731 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T20:45:15,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:45:15,793 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732653915731-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3bf6b9aeb9497b924c9a0719a31fc28a, NAME => 'testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732653915731', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 2024-11-26T20:45:15,800 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732653915731-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:15,800 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732653915731-pool-0 {}] regionserver.HRegion(1722): Closing 3bf6b9aeb9497b924c9a0719a31fc28a, disabling compactions & flushes 2024-11-26T20:45:15,800 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732653915731-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:15,800 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732653915731-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:15,800 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732653915731-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. after waiting 0 ms 2024-11-26T20:45:15,800 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732653915731-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:15,800 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732653915731-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:15,800 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732653915731-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3bf6b9aeb9497b924c9a0719a31fc28a: Waiting for close lock at 1732653915800Disabling compacts and flushes for region at 1732653915800Disabling writes for close at 1732653915800Writing region close event to WAL at 1732653915800Closed at 1732653915800 2024-11-26T20:45:15,801 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732653915731 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T20:45:15,806 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732653915802"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732653915802"}]},"ts":"1732653915802"} 2024-11-26T20:45:15,810 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T20:45:15,811 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732653915731 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T20:45:15,814 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732653915731","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653915811"}]},"ts":"1732653915811"} 2024-11-26T20:45:15,818 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732653915731, state=ENABLING in hbase:meta 2024-11-26T20:45:15,818 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {cd6be161543e=0} racks are {/default-rack=0} 2024-11-26T20:45:15,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T20:45:15,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T20:45:15,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T20:45:15,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T20:45:15,819 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T20:45:15,819 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T20:45:15,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T20:45:15,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, ASSIGN}] 2024-11-26T20:45:15,821 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, ASSIGN 2024-11-26T20:45:15,823 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, ASSIGN; state=OFFLINE, location=cd6be161543e,42887,1732653913730; forceNewPlan=false, retain=false 2024-11-26T20:45:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:45:15,976 INFO [cd6be161543e:45865 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T20:45:15,977 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3bf6b9aeb9497b924c9a0719a31fc28a, regionState=OPENING, regionLocation=cd6be161543e,42887,1732653913730 2024-11-26T20:45:15,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, ASSIGN because future has completed 2024-11-26T20:45:15,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bf6b9aeb9497b924c9a0719a31fc28a, server=cd6be161543e,42887,1732653913730}] 2024-11-26T20:45:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:45:16,138 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:16,138 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3bf6b9aeb9497b924c9a0719a31fc28a, NAME => 'testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:45:16,138 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732653915731 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,138 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:16,139 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,139 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,140 INFO [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,144 INFO [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bf6b9aeb9497b924c9a0719a31fc28a columnFamilyName family 2024-11-26T20:45:16,144 DEBUG [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:16,145 INFO [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] regionserver.HStore(327): Store=3bf6b9aeb9497b924c9a0719a31fc28a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:16,145 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,146 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,146 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,147 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,147 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,148 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,155 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:45:16,156 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3bf6b9aeb9497b924c9a0719a31fc28a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59151721, jitterRate=-0.11857067048549652}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:45:16,156 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:16,157 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3bf6b9aeb9497b924c9a0719a31fc28a: Running coprocessor pre-open hook at 1732653916139Writing region info on filesystem at 1732653916139Initializing all the Stores at 1732653916139Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653916139Cleaning up temporary data from old regions at 1732653916147 (+8 ms)Running coprocessor post-open hooks at 1732653916156 (+9 ms)Region opened successfully at 1732653916157 (+1 ms) 2024-11-26T20:45:16,158 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a., pid=6, masterSystemTime=1732653916134 2024-11-26T20:45:16,161 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3bf6b9aeb9497b924c9a0719a31fc28a, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,42887,1732653913730 2024-11-26T20:45:16,161 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:16,161 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:16,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bf6b9aeb9497b924c9a0719a31fc28a, server=cd6be161543e,42887,1732653913730 because future has completed 2024-11-26T20:45:16,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T20:45:16,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3bf6b9aeb9497b924c9a0719a31fc28a, server=cd6be161543e,42887,1732653913730 in 185 msec 2024-11-26T20:45:16,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T20:45:16,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, ASSIGN in 348 msec 2024-11-26T20:45:16,171 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732653915731 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T20:45:16,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732653915731","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653916171"}]},"ts":"1732653916171"} 2024-11-26T20:45:16,174 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732653915731, state=ENABLED in hbase:meta 2024-11-26T20:45:16,175 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732653915731 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T20:45:16,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732653915731 in 404 msec 2024-11-26T20:45:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:45:16,417 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732653915731 get assigned. Timeout = 60000ms 2024-11-26T20:45:16,417 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732653915731 completed 2024-11-26T20:45:16,419 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:45:16,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732653915731 assigned to meta. Checking AM states. 2024-11-26T20:45:16,425 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:45:16,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732653915731 assigned. 2024-11-26T20:45:16,434 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732653915731', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a., hostname=cd6be161543e,42887,1732653913730, seqNum=2] 2024-11-26T20:45:16,446 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3bf6b9aeb9497b924c9a0719a31fc28a 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-26T20:45:16,471 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 68dba4252f6e4d4388bc222846b531b4 2024-11-26T20:45:16,508 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/.tmp/family/68dba4252f6e4d4388bc222846b531b4 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-26T20:45:16,513 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/.tmp/family/68dba4252f6e4d4388bc222846b531b4 2024-11-26T20:45:16,549 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/.tmp/family/68dba4252f6e4d4388bc222846b531b4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4 2024-11-26T20:45:16,551 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4, delay=947 ms 2024-11-26T20:45:16,553 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4, entries=2, sequenceid=6, filesize=4.9 K 2024-11-26T20:45:16,558 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 3bf6b9aeb9497b924c9a0719a31fc28a in 113ms, sequenceid=6, compaction requested=false 2024-11-26T20:45:16,559 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732653915731' 2024-11-26T20:45:16,561 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3bf6b9aeb9497b924c9a0719a31fc28a: 2024-11-26T20:45:17,500 DEBUG [hfile-prefetch-1732653916551 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 68dba4252f6e4d4388bc222846b531b4_0. 2024-11-26T20:45:17,504 DEBUG [hfile-prefetch-1732653916551 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 68dba4252f6e4d4388bc222846b531b4_117. 2024-11-26T20:45:17,505 DEBUG [hfile-prefetch-1732653916551 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4, with total blocks 2, and data blocks 1 2024-11-26T20:45:17,505 DEBUG [hfile-prefetch-1732653916551 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T20:45:17,505 DEBUG [hfile-prefetch-1732653916551 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T20:45:17,506 DEBUG [hfile-prefetch-1732653916551 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4. Will sleep 100ms and try the verification again. 2024-11-26T20:45:17,550 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T20:45:17,550 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T20:45:17,551 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T20:45:17,551 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T20:45:17,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=3bf6b9aeb9497b924c9a0719a31fc28a, source=cd6be161543e,42887,1732653913730, destination=cd6be161543e,42287,1732653913595, warming up region on cd6be161543e,42287,1732653913595 2024-11-26T20:45:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T20:45:17,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=3bf6b9aeb9497b924c9a0719a31fc28a, source=cd6be161543e,42887,1732653913730, destination=cd6be161543e,42287,1732653913595, running balancer 2024-11-26T20:45:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, REOPEN/MOVE 2024-11-26T20:45:17,576 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T20:45:17,576 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, REOPEN/MOVE 2024-11-26T20:45:17,576 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732653917550 2024-11-26T20:45:17,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52839, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T20:45:17,577 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T20:45:17,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:17,578 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3bf6b9aeb9497b924c9a0719a31fc28a, regionState=CLOSING, regionLocation=cd6be161543e,42887,1732653913730 2024-11-26T20:45:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3bf6b9aeb9497b924c9a0719a31fc28a, NAME => 'testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:45:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:17,580 INFO [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:17,580 INFO [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bf6b9aeb9497b924c9a0719a31fc28a columnFamilyName family 2024-11-26T20:45:17,580 DEBUG [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:17,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, REOPEN/MOVE because future has completed 2024-11-26T20:45:17,581 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-26T20:45:17,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3bf6b9aeb9497b924c9a0719a31fc28a, server=cd6be161543e,42887,1732653913730}] 2024-11-26T20:45:17,585 DEBUG [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4 2024-11-26T20:45:17,585 INFO [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] regionserver.HStore(327): Store=3bf6b9aeb9497b924c9a0719a31fc28a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.HRegion(1722): Closing 3bf6b9aeb9497b924c9a0719a31fc28a, disabling compactions & flushes 2024-11-26T20:45:17,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. after waiting 0 ms 2024-11-26T20:45:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:17,587 DEBUG [StoreFileCloser-3bf6b9aeb9497b924c9a0719a31fc28a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4 2024-11-26T20:45:17,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42287 {}] regionserver.HRegion(1676): Region close journal for 3bf6b9aeb9497b924c9a0719a31fc28a: Waiting for close lock at 1732653917585Disabling compacts and flushes for region at 1732653917585Disabling writes for close at 1732653917586 (+1 ms)Writing region close event to WAL at 1732653917587 (+1 ms)Closed at 1732653917587 2024-11-26T20:45:17,588 WARN [hfile-prefetch-1732653916551 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-26T20:45:17,589 DEBUG [hfile-prefetch-1732653916551 {}] hfile.PrefetchExecutor(121): Prefetch completed for 68dba4252f6e4d4388bc222846b531b4 2024-11-26T20:45:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45865 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-26T20:45:17,740 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:17,740 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-26T20:45:17,740 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 3bf6b9aeb9497b924c9a0719a31fc28a, disabling compactions & flushes 2024-11-26T20:45:17,740 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:17,741 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:17,741 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. after waiting 0 ms 2024-11-26T20:45:17,741 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:17,742 DEBUG [StoreFileCloser-3bf6b9aeb9497b924c9a0719a31fc28a-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 68dba4252f6e4d4388bc222846b531b4, starting offset: 0, end offset: 9223372036854775807 2024-11-26T20:45:17,743 DEBUG [StoreFileCloser-3bf6b9aeb9497b924c9a0719a31fc28a-family-1 {}] bucket.BucketCache(856): removed key 68dba4252f6e4d4388bc222846b531b4_0 from back map with offset lock 2094080 in the evict process 2024-11-26T20:45:17,749 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T20:45:17,750 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:17,750 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 3bf6b9aeb9497b924c9a0719a31fc28a: Waiting for close lock at 1732653917740Running coprocessor pre-close hooks at 1732653917740Disabling compacts and flushes for region at 1732653917740Disabling writes for close at 1732653917741 (+1 ms)Writing region close event to WAL at 1732653917744 (+3 ms)Running coprocessor post-close hooks at 1732653917749 (+5 ms)Closed at 1732653917750 (+1 ms) 2024-11-26T20:45:17,750 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 3bf6b9aeb9497b924c9a0719a31fc28a move to cd6be161543e,42287,1732653913595 record at close sequenceid=6 2024-11-26T20:45:17,753 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:17,754 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3bf6b9aeb9497b924c9a0719a31fc28a, regionState=CLOSED 2024-11-26T20:45:17,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3bf6b9aeb9497b924c9a0719a31fc28a, server=cd6be161543e,42887,1732653913730 because future has completed 2024-11-26T20:45:17,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T20:45:17,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 3bf6b9aeb9497b924c9a0719a31fc28a, server=cd6be161543e,42887,1732653913730 in 175 msec 2024-11-26T20:45:17,760 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, REOPEN/MOVE; state=CLOSED, location=cd6be161543e,42287,1732653913595; forceNewPlan=false, retain=false 2024-11-26T20:45:17,777 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T20:45:17,777 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T20:45:17,777 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T20:45:17,777 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T20:45:17,781 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T20:45:17,781 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732653917777 2024-11-26T20:45:17,781 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T20:45:17,910 INFO [cd6be161543e:45865 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T20:45:17,910 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3bf6b9aeb9497b924c9a0719a31fc28a, regionState=OPENING, regionLocation=cd6be161543e,42287,1732653913595 2024-11-26T20:45:17,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, REOPEN/MOVE because future has completed 2024-11-26T20:45:17,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bf6b9aeb9497b924c9a0719a31fc28a, server=cd6be161543e,42287,1732653913595}] 2024-11-26T20:45:18,072 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:18,072 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3bf6b9aeb9497b924c9a0719a31fc28a, NAME => 'testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:45:18,073 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732653915731 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,073 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:18,073 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,073 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,074 INFO [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,075 INFO [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bf6b9aeb9497b924c9a0719a31fc28a columnFamilyName family 2024-11-26T20:45:18,075 DEBUG [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,077 DEBUG [StoreFileOpener-3bf6b9aeb9497b924c9a0719a31fc28a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4, delay=938 ms 2024-11-26T20:45:18,078 DEBUG [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4 2024-11-26T20:45:18,078 INFO [StoreOpener-3bf6b9aeb9497b924c9a0719a31fc28a-1 {}] regionserver.HStore(327): Store=3bf6b9aeb9497b924c9a0719a31fc28a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:18,078 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,079 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,079 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,080 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,080 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,084 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,085 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3bf6b9aeb9497b924c9a0719a31fc28a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72806938, jitterRate=0.08490791916847229}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:45:18,085 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,086 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3bf6b9aeb9497b924c9a0719a31fc28a: Running coprocessor pre-open hook at 1732653918073Writing region info on filesystem at 1732653918073Initializing all the Stores at 1732653918074 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653918074Cleaning up temporary data from old regions at 1732653918080 (+6 ms)Running coprocessor post-open hooks at 1732653918085 (+5 ms)Region opened successfully at 1732653918086 (+1 ms) 2024-11-26T20:45:18,087 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a., pid=9, masterSystemTime=1732653918069 2024-11-26T20:45:18,090 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:18,090 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:18,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3bf6b9aeb9497b924c9a0719a31fc28a, regionState=OPEN, openSeqNum=10, regionLocation=cd6be161543e,42287,1732653913595 2024-11-26T20:45:18,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bf6b9aeb9497b924c9a0719a31fc28a, server=cd6be161543e,42287,1732653913595 because future has completed 2024-11-26T20:45:18,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-26T20:45:18,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3bf6b9aeb9497b924c9a0719a31fc28a, server=cd6be161543e,42287,1732653913595 in 176 msec 2024-11-26T20:45:18,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732653915731, region=3bf6b9aeb9497b924c9a0719a31fc28a, REOPEN/MOVE in 520 msec 2024-11-26T20:45:18,129 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T20:45:18,129 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:45:18,129 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:45:18,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:18,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:18,134 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:45:18,134 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T20:45:18,134 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=632300009, stopped=false 2024-11-26T20:45:18,135 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=cd6be161543e,45865,1732653912827 2024-11-26T20:45:18,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:18,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:18,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:18,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:45:18,138 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:45:18,138 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:45:18,138 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:18,138 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:18,138 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:18,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:18,139 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cd6be161543e,42287,1732653913595' ***** 2024-11-26T20:45:18,139 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T20:45:18,139 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cd6be161543e,42887,1732653913730' ***** 2024-11-26T20:45:18,139 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T20:45:18,139 INFO [RS:0;cd6be161543e:42287 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T20:45:18,139 INFO [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T20:45:18,139 INFO [RS:0;cd6be161543e:42287 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.persistence 2024-11-26T20:45:18,140 INFO [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/bucket.persistence 2024-11-26T20:45:18,140 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T20:45:18,140 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T20:45:18,140 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T20:45:18,140 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T20:45:18,140 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T20:45:18,140 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T20:45:18,140 INFO [RS:0;cd6be161543e:42287 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T20:45:18,140 INFO [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T20:45:18,140 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T20:45:18,140 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T20:45:18,150 DEBUG [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1487): Thread RS:1;cd6be161543e:42887 started persisting bucket cache to file 2024-11-26T20:45:18,150 DEBUG [RS:0;cd6be161543e:42287 {}] bucket.BucketCache(1487): Thread RS:0;cd6be161543e:42287 started persisting bucket cache to file 2024-11-26T20:45:18,150 DEBUG [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T20:45:18,150 DEBUG [RS:0;cd6be161543e:42287 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T20:45:18,150 DEBUG [RS:0;cd6be161543e:42287 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T20:45:18,150 DEBUG [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T20:45:18,154 DEBUG [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T20:45:18,154 DEBUG [RS:0;cd6be161543e:42287 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T20:45:18,154 DEBUG [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732653918150 2024-11-26T20:45:18,154 DEBUG [RS:0;cd6be161543e:42287 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732653918150 2024-11-26T20:45:18,154 DEBUG [RS:0;cd6be161543e:42287 {}] bucket.BucketCache(1509): Thread RS:0;cd6be161543e:42287 finished persisting bucket cache to file, renaming 2024-11-26T20:45:18,154 DEBUG [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1509): Thread RS:1;cd6be161543e:42887 finished persisting bucket cache to file, renaming 2024-11-26T20:45:18,155 WARN [RS:1;cd6be161543e:42887 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-26T20:45:18,155 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T20:45:18,155 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T20:45:18,155 INFO [RS:0;cd6be161543e:42287 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T20:45:18,155 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T20:45:18,155 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T20:45:18,155 INFO [RS:1;cd6be161543e:42887 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T20:45:18,155 INFO [RS:0;cd6be161543e:42287 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T20:45:18,155 INFO [RS:1;cd6be161543e:42887 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T20:45:18,155 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(3091): Received CLOSE for 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,155 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(959): stopping server cd6be161543e,42887,1732653913730 2024-11-26T20:45:18,155 INFO [RS:1;cd6be161543e:42887 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:45:18,156 INFO [RS:1;cd6be161543e:42887 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;cd6be161543e:42887. 2024-11-26T20:45:18,156 DEBUG [RS:1;cd6be161543e:42887 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:45:18,156 DEBUG [RS:1;cd6be161543e:42887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:18,156 INFO [RS:1;cd6be161543e:42887 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T20:45:18,156 INFO [RS:1;cd6be161543e:42887 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T20:45:18,156 INFO [RS:1;cd6be161543e:42887 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T20:45:18,156 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(959): stopping server cd6be161543e,42287,1732653913595 2024-11-26T20:45:18,156 INFO [RS:0;cd6be161543e:42287 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:45:18,156 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T20:45:18,156 INFO [RS:0;cd6be161543e:42287 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;cd6be161543e:42287. 2024-11-26T20:45:18,156 DEBUG [RS:0;cd6be161543e:42287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:45:18,156 DEBUG [RS:0;cd6be161543e:42287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:18,156 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T20:45:18,157 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T20:45:18,156 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T20:45:18,157 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T20:45:18,157 DEBUG [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T20:45:18,157 DEBUG [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(1325): Online Regions={3bf6b9aeb9497b924c9a0719a31fc28a=testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a.} 2024-11-26T20:45:18,157 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3bf6b9aeb9497b924c9a0719a31fc28a, disabling compactions & flushes 2024-11-26T20:45:18,157 DEBUG [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:45:18,157 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T20:45:18,157 DEBUG [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(1351): Waiting on 3bf6b9aeb9497b924c9a0719a31fc28a 2024-11-26T20:45:18,157 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:18,157 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T20:45:18,157 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:18,157 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T20:45:18,157 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. after waiting 0 ms 2024-11-26T20:45:18,157 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T20:45:18,157 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:18,157 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T20:45:18,157 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-26T20:45:18,160 DEBUG [StoreFileCloser-3bf6b9aeb9497b924c9a0719a31fc28a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/family/68dba4252f6e4d4388bc222846b531b4 2024-11-26T20:45:18,167 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnRegionMove1732653915731/3bf6b9aeb9497b924c9a0719a31fc28a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-26T20:45:18,167 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:18,167 INFO [regionserver/cd6be161543e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:18,167 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3bf6b9aeb9497b924c9a0719a31fc28a: Waiting for close lock at 1732653918156Running coprocessor pre-close hooks at 1732653918157 (+1 ms)Disabling compacts and flushes for region at 1732653918157Disabling writes for close at 1732653918157Writing region close event to WAL at 1732653918160 (+3 ms)Running coprocessor post-close hooks at 1732653918167 (+7 ms)Closed at 1732653918167 2024-11-26T20:45:18,167 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a. 2024-11-26T20:45:18,170 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3fef6f65fc554b508801668b8885d335 2024-11-26T20:45:18,188 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/info/3fef6f65fc554b508801668b8885d335 is 205, key is testBlockEvictionOnRegionMove1732653915731,,1732653915765.3bf6b9aeb9497b924c9a0719a31fc28a./info:regioninfo/1732653918090/Put/seqid=0 2024-11-26T20:45:18,189 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/info/3fef6f65fc554b508801668b8885d335 2024-11-26T20:45:18,199 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b03452b38b5d471f80c2106457e96f56 2024-11-26T20:45:18,211 INFO [regionserver/cd6be161543e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:18,213 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/ns/b03452b38b5d471f80c2106457e96f56 is 43, key is default/ns:d/1732653915570/Put/seqid=0 2024-11-26T20:45:18,214 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/ns/b03452b38b5d471f80c2106457e96f56 2024-11-26T20:45:18,223 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5c41e2045f2b458ba16fb2ba503a8898 2024-11-26T20:45:18,236 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/table/5c41e2045f2b458ba16fb2ba503a8898 is 78, key is testBlockEvictionOnRegionMove1732653915731/table:state/1732653916171/Put/seqid=0 2024-11-26T20:45:18,236 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/table/5c41e2045f2b458ba16fb2ba503a8898 2024-11-26T20:45:18,239 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/info/3fef6f65fc554b508801668b8885d335 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/info/3fef6f65fc554b508801668b8885d335 2024-11-26T20:45:18,240 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/info/3fef6f65fc554b508801668b8885d335, delay=904 ms 2024-11-26T20:45:18,241 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/info/3fef6f65fc554b508801668b8885d335, entries=15, sequenceid=15, filesize=7.6 K 2024-11-26T20:45:18,242 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/ns/b03452b38b5d471f80c2106457e96f56 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/ns/b03452b38b5d471f80c2106457e96f56 2024-11-26T20:45:18,243 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/ns/b03452b38b5d471f80c2106457e96f56, delay=970 ms 2024-11-26T20:45:18,244 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/ns/b03452b38b5d471f80c2106457e96f56, entries=2, sequenceid=15, filesize=5.0 K 2024-11-26T20:45:18,245 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/table/5c41e2045f2b458ba16fb2ba503a8898 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/table/5c41e2045f2b458ba16fb2ba503a8898 2024-11-26T20:45:18,246 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/table/5c41e2045f2b458ba16fb2ba503a8898, delay=911 ms 2024-11-26T20:45:18,247 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/table/5c41e2045f2b458ba16fb2ba503a8898, entries=2, sequenceid=15, filesize=5.3 K 2024-11-26T20:45:18,248 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false 2024-11-26T20:45:18,248 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T20:45:18,249 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/info/3fef6f65fc554b508801668b8885d335 2024-11-26T20:45:18,250 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/ns/b03452b38b5d471f80c2106457e96f56 2024-11-26T20:45:18,250 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/table/5c41e2045f2b458ba16fb2ba503a8898 2024-11-26T20:45:18,255 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-26T20:45:18,255 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T20:45:18,255 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T20:45:18,256 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732653918156Running coprocessor pre-close hooks at 1732653918157 (+1 ms)Disabling compacts and flushes for region at 1732653918157Disabling writes for close at 1732653918157Obtaining lock to block concurrent updates at 1732653918157Preparing flush snapshotting stores in 1588230740 at 1732653918158 (+1 ms)Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732653918158Flushing stores of hbase:meta,,1.1588230740 at 1732653918158Flushing 1588230740/info: creating writer at 1732653918158Flushing 1588230740/info: appending metadata at 1732653918187 (+29 ms)Flushing 1588230740/info: closing flushed file at 1732653918187Flushing 1588230740/ns: creating writer at 1732653918192 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732653918212 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732653918212Flushing 1588230740/table: creating writer at 1732653918217 (+5 ms)Flushing 1588230740/table: appending metadata at 1732653918235 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732653918235Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cd78472: reopening flushed file at 1732653918239 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@247c3971: reopening flushed file at 1732653918242 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35420eff: reopening flushed file at 1732653918244 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false at 1732653918248 (+4 ms)Writing region close event to WAL at 1732653918251 (+3 ms)Running coprocessor post-close hooks at 1732653918255 (+4 ms)Closed at 1732653918255 2024-11-26T20:45:18,256 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T20:45:18,357 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(976): stopping server cd6be161543e,42287,1732653913595; all regions closed. 2024-11-26T20:45:18,357 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(976): stopping server cd6be161543e,42887,1732653913730; all regions closed. 2024-11-26T20:45:18,363 DEBUG [RS:1;cd6be161543e:42887 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs 2024-11-26T20:45:18,363 DEBUG [RS:0;cd6be161543e:42287 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs 2024-11-26T20:45:18,363 INFO [RS:1;cd6be161543e:42887 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cd6be161543e%2C42887%2C1732653913730.meta:.meta(num 1732653915433) 2024-11-26T20:45:18,363 INFO [RS:0;cd6be161543e:42287 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cd6be161543e%2C42287%2C1732653913595:(num 1732653915008) 2024-11-26T20:45:18,363 DEBUG [RS:0;cd6be161543e:42287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:18,363 INFO [RS:0;cd6be161543e:42287 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:18,363 INFO [RS:0;cd6be161543e:42287 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:45:18,363 INFO [RS:0;cd6be161543e:42287 {}] hbase.ChoreService(370): Chore service for: regionserver/cd6be161543e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T20:45:18,364 INFO [RS:0;cd6be161543e:42287 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T20:45:18,364 INFO [regionserver/cd6be161543e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:45:18,364 INFO [RS:0;cd6be161543e:42287 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T20:45:18,364 INFO [RS:0;cd6be161543e:42287 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T20:45:18,364 INFO [RS:0;cd6be161543e:42287 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:45:18,364 INFO [RS:0;cd6be161543e:42287 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42287 2024-11-26T20:45:18,365 DEBUG [RS:1;cd6be161543e:42887 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs 2024-11-26T20:45:18,365 INFO [RS:1;cd6be161543e:42887 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cd6be161543e%2C42887%2C1732653913730:(num 1732653914964) 2024-11-26T20:45:18,365 DEBUG [RS:1;cd6be161543e:42887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:18,365 INFO [RS:1;cd6be161543e:42887 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:18,365 INFO [RS:1;cd6be161543e:42887 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:45:18,365 INFO [RS:1;cd6be161543e:42887 {}] hbase.ChoreService(370): Chore service for: regionserver/cd6be161543e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T20:45:18,365 INFO [RS:1;cd6be161543e:42887 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:45:18,365 INFO [regionserver/cd6be161543e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:45:18,365 INFO [RS:1;cd6be161543e:42887 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42887 2024-11-26T20:45:18,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cd6be161543e,42887,1732653913730 2024-11-26T20:45:18,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:45:18,369 INFO [RS:1;cd6be161543e:42887 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:45:18,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cd6be161543e,42287,1732653913595 2024-11-26T20:45:18,371 INFO [RS:0;cd6be161543e:42287 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:45:18,373 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cd6be161543e,42887,1732653913730] 2024-11-26T20:45:18,374 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cd6be161543e,42887,1732653913730 already deleted, retry=false 2024-11-26T20:45:18,374 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cd6be161543e,42887,1732653913730 expired; onlineServers=1 2024-11-26T20:45:18,374 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cd6be161543e,42287,1732653913595] 2024-11-26T20:45:18,376 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cd6be161543e,42287,1732653913595 already deleted, retry=false 2024-11-26T20:45:18,376 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cd6be161543e,42287,1732653913595 expired; onlineServers=0 2024-11-26T20:45:18,376 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'cd6be161543e,45865,1732653912827' ***** 2024-11-26T20:45:18,376 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T20:45:18,376 INFO [M:0;cd6be161543e:45865 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:45:18,376 INFO [M:0;cd6be161543e:45865 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:45:18,376 DEBUG [M:0;cd6be161543e:45865 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T20:45:18,377 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T20:45:18,377 DEBUG [M:0;cd6be161543e:45865 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T20:45:18,377 DEBUG [master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.large.0-1732653914699 {}] cleaner.HFileCleaner(306): Exit Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.large.0-1732653914699,5,FailOnTimeoutGroup] 2024-11-26T20:45:18,377 DEBUG [master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.small.0-1732653914700 {}] cleaner.HFileCleaner(306): Exit Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.small.0-1732653914700,5,FailOnTimeoutGroup] 2024-11-26T20:45:18,377 INFO [M:0;cd6be161543e:45865 {}] hbase.ChoreService(370): Chore service for: master/cd6be161543e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T20:45:18,377 INFO [M:0;cd6be161543e:45865 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:45:18,377 DEBUG [M:0;cd6be161543e:45865 {}] master.HMaster(1795): Stopping service threads 2024-11-26T20:45:18,377 INFO [M:0;cd6be161543e:45865 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T20:45:18,377 INFO [M:0;cd6be161543e:45865 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:45:18,378 INFO [M:0;cd6be161543e:45865 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:45:18,378 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T20:45:18,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T20:45:18,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,380 DEBUG [M:0;cd6be161543e:45865 {}] zookeeper.ZKUtil(347): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T20:45:18,380 WARN [M:0;cd6be161543e:45865 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T20:45:18,380 INFO [M:0;cd6be161543e:45865 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/.lastflushedseqids 2024-11-26T20:45:18,389 INFO [M:0;cd6be161543e:45865 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:45:18,389 INFO [M:0;cd6be161543e:45865 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T20:45:18,390 DEBUG [M:0;cd6be161543e:45865 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T20:45:18,390 INFO [M:0;cd6be161543e:45865 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:18,390 DEBUG [M:0;cd6be161543e:45865 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:18,390 DEBUG [M:0;cd6be161543e:45865 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T20:45:18,390 DEBUG [M:0;cd6be161543e:45865 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:18,390 INFO [M:0;cd6be161543e:45865 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-26T20:45:18,400 DEBUG [M:0;cd6be161543e:45865 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 126c5871c40d4023b18744eb4b474501 2024-11-26T20:45:18,414 DEBUG [M:0;cd6be161543e:45865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/126c5871c40d4023b18744eb4b474501 is 82, key is hbase:meta,,1/info:regioninfo/1732653915503/Put/seqid=0 2024-11-26T20:45:18,415 INFO [M:0;cd6be161543e:45865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/126c5871c40d4023b18744eb4b474501 2024-11-26T20:45:18,425 DEBUG [M:0;cd6be161543e:45865 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cae3e27b17be40d0805a365508a80fcf 2024-11-26T20:45:18,440 DEBUG [M:0;cd6be161543e:45865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cae3e27b17be40d0805a365508a80fcf is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732653916176/Put/seqid=0 2024-11-26T20:45:18,441 INFO [M:0;cd6be161543e:45865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cae3e27b17be40d0805a365508a80fcf 2024-11-26T20:45:18,450 DEBUG [M:0;cd6be161543e:45865 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7c50dd4a18d94f9d854dc4f47db7d5fa 2024-11-26T20:45:18,463 DEBUG [M:0;cd6be161543e:45865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c50dd4a18d94f9d854dc4f47db7d5fa is 69, key is cd6be161543e,42287,1732653913595/rs:state/1732653914839/Put/seqid=0 2024-11-26T20:45:18,464 INFO [M:0;cd6be161543e:45865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c50dd4a18d94f9d854dc4f47db7d5fa 2024-11-26T20:45:18,466 DEBUG [M:0;cd6be161543e:45865 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/126c5871c40d4023b18744eb4b474501 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/126c5871c40d4023b18744eb4b474501 2024-11-26T20:45:18,468 INFO [M:0;cd6be161543e:45865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/126c5871c40d4023b18744eb4b474501, entries=8, sequenceid=84, filesize=5.5 K 2024-11-26T20:45:18,469 DEBUG [M:0;cd6be161543e:45865 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cae3e27b17be40d0805a365508a80fcf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cae3e27b17be40d0805a365508a80fcf 2024-11-26T20:45:18,471 INFO [M:0;cd6be161543e:45865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cae3e27b17be40d0805a365508a80fcf, entries=9, sequenceid=84, filesize=6.5 K 2024-11-26T20:45:18,472 DEBUG [M:0;cd6be161543e:45865 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c50dd4a18d94f9d854dc4f47db7d5fa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7c50dd4a18d94f9d854dc4f47db7d5fa 2024-11-26T20:45:18,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:18,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42887-0x10156067c6d0002, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:18,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:18,474 INFO [RS:1;cd6be161543e:42887 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:45:18,474 INFO [RS:0;cd6be161543e:42287 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:45:18,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42287-0x10156067c6d0001, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:18,474 INFO [M:0;cd6be161543e:45865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7c50dd4a18d94f9d854dc4f47db7d5fa, entries=2, sequenceid=84, filesize=5.1 K 2024-11-26T20:45:18,474 INFO [RS:1;cd6be161543e:42887 {}] regionserver.HRegionServer(1031): Exiting; stopping=cd6be161543e,42887,1732653913730; zookeeper connection closed. 2024-11-26T20:45:18,474 INFO [RS:0;cd6be161543e:42287 {}] regionserver.HRegionServer(1031): Exiting; stopping=cd6be161543e,42287,1732653913595; zookeeper connection closed. 2024-11-26T20:45:18,474 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@56fbbd16 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@56fbbd16 2024-11-26T20:45:18,474 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@35791dc8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@35791dc8 2024-11-26T20:45:18,475 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-26T20:45:18,475 INFO [M:0;cd6be161543e:45865 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false 2024-11-26T20:45:18,476 INFO [M:0;cd6be161543e:45865 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:18,476 DEBUG [M:0;cd6be161543e:45865 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732653918389Disabling compacts and flushes for region at 1732653918389Disabling writes for close at 1732653918390 (+1 ms)Obtaining lock to block concurrent updates at 1732653918390Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732653918390Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1732653918390Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732653918391 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732653918391Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732653918414 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732653918414Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732653918418 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732653918439 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732653918439Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732653918443 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732653918462 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732653918463 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@107cf858: reopening flushed file at 1732653918466 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c4ba47f: reopening flushed file at 1732653918468 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a98b7f8: reopening flushed file at 1732653918471 (+3 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false at 1732653918475 (+4 ms)Writing region close event to WAL at 1732653918476 (+1 ms)Closed at 1732653918476 2024-11-26T20:45:18,478 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:45:18,478 INFO [M:0;cd6be161543e:45865 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T20:45:18,478 INFO [M:0;cd6be161543e:45865 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45865 2024-11-26T20:45:18,479 INFO [M:0;cd6be161543e:45865 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:45:18,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:18,581 INFO [M:0;cd6be161543e:45865 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:45:18,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45865-0x10156067c6d0000, quorum=127.0.0.1:49272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:18,585 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T20:45:18,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T20:45:18,606 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:18,614 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/cd6be161543e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732653918240 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732653918077 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732653916551 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/cd6be161543e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1732653918243 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/cd6be161543e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/cd6be161543e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=386 (was 419), ProcessCount=11 (was 11), AvailableMemoryMB=6113 (was 6451) 2024-11-26T20:45:18,619 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=386, ProcessCount=11, AvailableMemoryMB=6113 2024-11-26T20:45:18,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/hadoop-log-dir so I do NOT create it in target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b 2024-11-26T20:45:18,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/hadoop-tmp-dir so I do NOT create it in target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b 2024-11-26T20:45:18,620 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:18,623 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/cluster_06589e4f-3782-c36a-49bd-fc2541aa8e0f, deleteOnExit=true 2024-11-26T20:45:18,625 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/cluster_06589e4f-3782-c36a-49bd-fc2541aa8e0f/zookeeper_0, clientPort=54450, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/cluster_06589e4f-3782-c36a-49bd-fc2541aa8e0f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/cluster_06589e4f-3782-c36a-49bd-fc2541aa8e0f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T20:45:18,626 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54450 2024-11-26T20:45:18,638 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 with version=8 2024-11-26T20:45:18,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/hbase-staging 2024-11-26T20:45:18,642 INFO [Time-limited test {}] client.ConnectionUtils(128): master/cd6be161543e:0 server-side Connection retries=45 2024-11-26T20:45:18,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:18,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:18,642 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:45:18,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:18,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:45:18,642 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T20:45:18,642 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:45:18,643 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46331 2024-11-26T20:45:18,644 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46331 connecting to ZooKeeper ensemble=127.0.0.1:54450 2024-11-26T20:45:18,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:463310x0, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:45:18,651 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46331-0x101560694310000 connected 2024-11-26T20:45:18,668 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:18,668 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:18,669 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:18,669 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31, hbase.cluster.distributed=false 2024-11-26T20:45:18,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:45:18,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46331 2024-11-26T20:45:18,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46331 2024-11-26T20:45:18,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46331 2024-11-26T20:45:18,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46331 2024-11-26T20:45:18,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46331 2024-11-26T20:45:18,691 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cd6be161543e:0 server-side Connection retries=45 2024-11-26T20:45:18,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:18,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:18,691 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:45:18,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:18,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:45:18,691 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T20:45:18,691 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:45:18,692 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37675 2024-11-26T20:45:18,693 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37675 connecting to ZooKeeper ensemble=127.0.0.1:54450 2024-11-26T20:45:18,693 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:18,694 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:18,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376750x0, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:45:18,697 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:18,697 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37675-0x101560694310001 connected 2024-11-26T20:45:18,698 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T20:45:18,698 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.cache 2024-11-26T20:45:18,698 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T20:45:18,699 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T20:45:18,699 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T20:45:18,699 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T20:45:18,699 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T20:45:18,700 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T20:45:18,700 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.persistence is complete. 2024-11-26T20:45:18,700 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T20:45:18,702 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T20:45:18,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T20:45:18,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:45:18,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37675 2024-11-26T20:45:18,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37675 2024-11-26T20:45:18,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37675 2024-11-26T20:45:18,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37675 2024-11-26T20:45:18,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37675 2024-11-26T20:45:18,727 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cd6be161543e:0 server-side Connection retries=45 2024-11-26T20:45:18,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:18,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:18,727 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:45:18,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:18,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:45:18,727 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T20:45:18,727 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:45:18,728 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46687 2024-11-26T20:45:18,729 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46687 connecting to ZooKeeper ensemble=127.0.0.1:54450 2024-11-26T20:45:18,730 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:18,730 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:18,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466870x0, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:45:18,733 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:18,733 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46687-0x101560694310002 connected 2024-11-26T20:45:18,734 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T20:45:18,734 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.cache 2024-11-26T20:45:18,734 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T20:45:18,734 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T20:45:18,734 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T20:45:18,735 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T20:45:18,735 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T20:45:18,735 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T20:45:18,735 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T20:45:18,736 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.persistence is complete. 2024-11-26T20:45:18,736 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T20:45:18,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T20:45:18,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:45:18,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46687 2024-11-26T20:45:18,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46687 2024-11-26T20:45:18,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46687 2024-11-26T20:45:18,744 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46687 2024-11-26T20:45:18,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46687 2024-11-26T20:45:18,756 DEBUG [M:0;cd6be161543e:46331 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cd6be161543e:46331 2024-11-26T20:45:18,756 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cd6be161543e,46331,1732653918641 2024-11-26T20:45:18,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:18,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:18,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:18,759 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cd6be161543e,46331,1732653918641 2024-11-26T20:45:18,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T20:45:18,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T20:45:18,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,763 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T20:45:18,764 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cd6be161543e,46331,1732653918641 from backup master directory 2024-11-26T20:45:18,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cd6be161543e,46331,1732653918641 2024-11-26T20:45:18,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:18,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:18,765 WARN [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:45:18,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:45:18,765 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cd6be161543e,46331,1732653918641 2024-11-26T20:45:18,767 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/hbase.id] with ID: dea3817a-6925-49d9-8e0f-4f6639213d6c 2024-11-26T20:45:18,767 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/.tmp/hbase.id 2024-11-26T20:45:18,771 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/hbase.id] 2024-11-26T20:45:18,809 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:18,809 INFO [master/cd6be161543e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T20:45:18,811 INFO [master/cd6be161543e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-26T20:45:18,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,818 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:45:18,819 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T20:45:18,819 INFO [master/cd6be161543e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:45:18,840 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store 2024-11-26T20:45:18,846 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:18,846 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T20:45:18,846 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:18,846 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:18,846 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T20:45:18,846 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:18,846 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:18,847 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732653918846Disabling compacts and flushes for region at 1732653918846Disabling writes for close at 1732653918846Writing region close event to WAL at 1732653918846Closed at 1732653918846 2024-11-26T20:45:18,847 WARN [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/.initializing 2024-11-26T20:45:18,847 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/WALs/cd6be161543e,46331,1732653918641 2024-11-26T20:45:18,852 INFO [master/cd6be161543e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:45:18,856 INFO [master/cd6be161543e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cd6be161543e%2C46331%2C1732653918641, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/WALs/cd6be161543e,46331,1732653918641, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/oldWALs, maxLogs=10 2024-11-26T20:45:18,874 INFO [master/cd6be161543e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/WALs/cd6be161543e,46331,1732653918641/cd6be161543e%2C46331%2C1732653918641.1732653918856 2024-11-26T20:45:18,874 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T20:45:18,874 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:45:18,874 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:18,874 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,874 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T20:45:18,880 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:18,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T20:45:18,884 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:18,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T20:45:18,888 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:18,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,891 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T20:45:18,891 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,891 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:18,892 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,892 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,892 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,892 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,892 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,893 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:45:18,893 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:45:18,899 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:45:18,899 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62804968, jitterRate=-0.06413304805755615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:45:18,900 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732653918874Initializing all the Stores at 1732653918875 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653918875Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653918875Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653918875Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653918875Cleaning up temporary data from old regions at 1732653918892 (+17 ms)Region opened successfully at 1732653918900 (+8 ms) 2024-11-26T20:45:18,900 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T20:45:18,904 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444a83c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cd6be161543e/172.17.0.2:0 2024-11-26T20:45:18,905 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T20:45:18,905 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T20:45:18,905 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T20:45:18,906 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:45:18,906 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:45:18,906 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T20:45:18,906 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T20:45:18,908 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T20:45:18,909 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T20:45:18,911 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T20:45:18,911 INFO [master/cd6be161543e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T20:45:18,912 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T20:45:18,914 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T20:45:18,914 INFO [master/cd6be161543e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T20:45:18,915 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T20:45:18,916 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T20:45:18,917 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T20:45:18,918 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T20:45:18,921 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T20:45:18,922 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T20:45:18,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:18,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:18,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:18,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,926 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=cd6be161543e,46331,1732653918641, sessionid=0x101560694310000, setting cluster-up flag (Was=false) 2024-11-26T20:45:18,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,935 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T20:45:18,936 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cd6be161543e,46331,1732653918641 2024-11-26T20:45:18,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:18,944 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T20:45:18,945 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cd6be161543e,46331,1732653918641 2024-11-26T20:45:18,946 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T20:45:18,948 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:45:18,948 INFO [master/cd6be161543e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T20:45:18,948 INFO [master/cd6be161543e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T20:45:18,949 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: cd6be161543e,46331,1732653918641 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T20:45:18,950 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:45:18,950 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:45:18,950 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:45:18,951 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:45:18,951 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/cd6be161543e:0, corePoolSize=10, maxPoolSize=10 2024-11-26T20:45:18,951 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:18,951 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/cd6be161543e:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:45:18,951 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:18,952 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:45:18,953 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732653948952 2024-11-26T20:45:18,953 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T20:45:18,953 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T20:45:18,953 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T20:45:18,953 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T20:45:18,953 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T20:45:18,953 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T20:45:18,953 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T20:45:18,953 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,953 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:18,953 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T20:45:18,954 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T20:45:18,954 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T20:45:18,954 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T20:45:18,957 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T20:45:18,957 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T20:45:18,957 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.large.0-1732653918957,5,FailOnTimeoutGroup] 2024-11-26T20:45:18,960 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.small.0-1732653918957,5,FailOnTimeoutGroup] 2024-11-26T20:45:18,960 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:18,960 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T20:45:18,961 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:18,961 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:18,967 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T20:45:18,967 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 2024-11-26T20:45:18,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:18,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T20:45:18,977 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T20:45:18,977 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:18,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T20:45:18,981 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T20:45:18,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:18,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T20:45:18,985 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T20:45:18,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:18,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T20:45:18,989 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T20:45:18,989 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:18,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:18,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T20:45:18,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740 2024-11-26T20:45:18,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740 2024-11-26T20:45:18,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T20:45:18,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T20:45:18,990 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:45:18,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T20:45:18,997 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:45:18,997 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60458910, jitterRate=-0.099092036485672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:45:18,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732653918973Initializing all the Stores at 1732653918974 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653918974Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653918974Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653918974Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653918974Cleaning up temporary data from old regions at 1732653918990 (+16 ms)Region opened successfully at 1732653918997 (+7 ms) 2024-11-26T20:45:18,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T20:45:18,998 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T20:45:18,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T20:45:18,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T20:45:18,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T20:45:18,998 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T20:45:18,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732653918998Disabling compacts and flushes for region at 1732653918998Disabling writes for close at 1732653918998Writing region close event to WAL at 1732653918998Closed at 1732653918998 2024-11-26T20:45:18,999 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:45:18,999 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T20:45:19,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T20:45:19,001 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:45:19,002 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T20:45:19,047 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(746): ClusterId : dea3817a-6925-49d9-8e0f-4f6639213d6c 2024-11-26T20:45:19,047 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(746): ClusterId : dea3817a-6925-49d9-8e0f-4f6639213d6c 2024-11-26T20:45:19,047 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T20:45:19,047 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T20:45:19,050 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T20:45:19,050 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T20:45:19,050 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T20:45:19,050 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T20:45:19,053 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T20:45:19,053 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T20:45:19,053 DEBUG [RS:1;cd6be161543e:46687 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d381ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cd6be161543e/172.17.0.2:0 2024-11-26T20:45:19,053 DEBUG [RS:0;cd6be161543e:37675 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0f8da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cd6be161543e/172.17.0.2:0 2024-11-26T20:45:19,067 DEBUG [RS:0;cd6be161543e:37675 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;cd6be161543e:37675 2024-11-26T20:45:19,067 INFO [RS:0;cd6be161543e:37675 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T20:45:19,067 INFO [RS:0;cd6be161543e:37675 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T20:45:19,067 DEBUG [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T20:45:19,068 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(2659): reportForDuty to master=cd6be161543e,46331,1732653918641 with port=37675, startcode=1732653918690 2024-11-26T20:45:19,068 DEBUG [RS:0;cd6be161543e:37675 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T20:45:19,070 DEBUG [RS:1;cd6be161543e:46687 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;cd6be161543e:46687 2024-11-26T20:45:19,070 INFO [RS:1;cd6be161543e:46687 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T20:45:19,070 INFO [RS:1;cd6be161543e:46687 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T20:45:19,070 DEBUG [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T20:45:19,071 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51431, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T20:45:19,071 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(2659): reportForDuty to master=cd6be161543e,46331,1732653918641 with port=46687, startcode=1732653918727 2024-11-26T20:45:19,071 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cd6be161543e,37675,1732653918690 2024-11-26T20:45:19,071 DEBUG [RS:1;cd6be161543e:46687 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T20:45:19,072 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46331 {}] master.ServerManager(517): Registering regionserver=cd6be161543e,37675,1732653918690 2024-11-26T20:45:19,073 DEBUG [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 2024-11-26T20:45:19,073 DEBUG [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T20:45:19,073 DEBUG [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T20:45:19,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:45:19,076 DEBUG [RS:0;cd6be161543e:37675 {}] zookeeper.ZKUtil(111): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cd6be161543e,37675,1732653918690 2024-11-26T20:45:19,076 WARN [RS:0;cd6be161543e:37675 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:45:19,076 INFO [RS:0;cd6be161543e:37675 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:45:19,076 DEBUG [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,37675,1732653918690 2024-11-26T20:45:19,077 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cd6be161543e,37675,1732653918690] 2024-11-26T20:45:19,079 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49753, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T20:45:19,079 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cd6be161543e,46687,1732653918727 2024-11-26T20:45:19,079 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46331 {}] master.ServerManager(517): Registering regionserver=cd6be161543e,46687,1732653918727 2024-11-26T20:45:19,081 DEBUG [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 2024-11-26T20:45:19,081 DEBUG [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T20:45:19,082 DEBUG [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T20:45:19,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:45:19,087 DEBUG [RS:1;cd6be161543e:46687 {}] zookeeper.ZKUtil(111): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cd6be161543e,46687,1732653918727 2024-11-26T20:45:19,087 WARN [RS:1;cd6be161543e:46687 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:45:19,087 INFO [RS:1;cd6be161543e:46687 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:45:19,087 DEBUG [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,46687,1732653918727 2024-11-26T20:45:19,088 INFO [RS:0;cd6be161543e:37675 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T20:45:19,088 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cd6be161543e,46687,1732653918727] 2024-11-26T20:45:19,090 INFO [RS:0;cd6be161543e:37675 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T20:45:19,092 INFO [RS:0;cd6be161543e:37675 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T20:45:19,092 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,093 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T20:45:19,093 INFO [RS:0;cd6be161543e:37675 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T20:45:19,094 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,094 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,094 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,094 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,094 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,094 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,094 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cd6be161543e:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:45:19,095 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,095 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,095 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,095 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,095 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,095 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,095 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:19,095 DEBUG [RS:0;cd6be161543e:37675 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:19,095 INFO [RS:1;cd6be161543e:46687 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T20:45:19,096 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,096 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,096 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,096 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,096 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,096 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,37675,1732653918690-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:45:19,098 INFO [RS:1;cd6be161543e:46687 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T20:45:19,098 INFO [RS:1;cd6be161543e:46687 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T20:45:19,098 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,100 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T20:45:19,101 INFO [RS:1;cd6be161543e:46687 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T20:45:19,101 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,101 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,101 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,101 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,101 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,101 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,101 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cd6be161543e:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:45:19,101 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,102 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,102 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,102 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,102 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,102 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:19,102 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:19,102 DEBUG [RS:1;cd6be161543e:46687 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:19,102 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,102 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,102 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,102 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,102 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,103 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,46687,1732653918727-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:45:19,114 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T20:45:19,114 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,37675,1732653918690-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,114 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,115 INFO [RS:0;cd6be161543e:37675 {}] regionserver.Replication(171): cd6be161543e,37675,1732653918690 started 2024-11-26T20:45:19,117 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T20:45:19,117 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,46687,1732653918727-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,118 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,118 INFO [RS:1;cd6be161543e:46687 {}] regionserver.Replication(171): cd6be161543e,46687,1732653918727 started 2024-11-26T20:45:19,129 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,129 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(1482): Serving as cd6be161543e,37675,1732653918690, RpcServer on cd6be161543e/172.17.0.2:37675, sessionid=0x101560694310001 2024-11-26T20:45:19,129 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T20:45:19,129 DEBUG [RS:0;cd6be161543e:37675 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cd6be161543e,37675,1732653918690 2024-11-26T20:45:19,129 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,37675,1732653918690' 2024-11-26T20:45:19,129 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T20:45:19,130 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T20:45:19,131 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T20:45:19,131 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T20:45:19,131 DEBUG [RS:0;cd6be161543e:37675 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cd6be161543e,37675,1732653918690 2024-11-26T20:45:19,131 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,37675,1732653918690' 2024-11-26T20:45:19,131 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T20:45:19,131 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T20:45:19,132 DEBUG [RS:0;cd6be161543e:37675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T20:45:19,132 INFO [RS:0;cd6be161543e:37675 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T20:45:19,132 INFO [RS:0;cd6be161543e:37675 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T20:45:19,135 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,135 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(1482): Serving as cd6be161543e,46687,1732653918727, RpcServer on cd6be161543e/172.17.0.2:46687, sessionid=0x101560694310002 2024-11-26T20:45:19,135 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T20:45:19,135 DEBUG [RS:1;cd6be161543e:46687 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cd6be161543e,46687,1732653918727 2024-11-26T20:45:19,135 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,46687,1732653918727' 2024-11-26T20:45:19,135 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T20:45:19,136 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T20:45:19,136 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T20:45:19,136 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T20:45:19,136 DEBUG [RS:1;cd6be161543e:46687 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cd6be161543e,46687,1732653918727 2024-11-26T20:45:19,137 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,46687,1732653918727' 2024-11-26T20:45:19,137 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T20:45:19,137 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T20:45:19,137 DEBUG [RS:1;cd6be161543e:46687 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T20:45:19,138 INFO [RS:1;cd6be161543e:46687 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T20:45:19,138 INFO [RS:1;cd6be161543e:46687 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T20:45:19,152 WARN [cd6be161543e:46331 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T20:45:19,233 INFO [RS:0;cd6be161543e:37675 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:45:19,233 INFO [RS:0;cd6be161543e:37675 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=cd6be161543e%2C37675%2C1732653918690, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,37675,1732653918690, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs, maxLogs=55 2024-11-26T20:45:19,238 INFO [RS:1;cd6be161543e:46687 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:45:19,239 INFO [RS:1;cd6be161543e:46687 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=cd6be161543e%2C46687%2C1732653918727, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,46687,1732653918727, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs, maxLogs=55 2024-11-26T20:45:19,253 INFO [RS:0;cd6be161543e:37675 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,37675,1732653918690/cd6be161543e%2C37675%2C1732653918690.1732653919235 2024-11-26T20:45:19,254 DEBUG [RS:0;cd6be161543e:37675 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T20:45:19,259 INFO [RS:1;cd6be161543e:46687 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,46687,1732653918727/cd6be161543e%2C46687%2C1732653918727.1732653919239 2024-11-26T20:45:19,259 DEBUG [RS:1;cd6be161543e:46687 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T20:45:19,402 DEBUG [cd6be161543e:46331 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T20:45:19,402 DEBUG [cd6be161543e:46331 {}] balancer.BalancerClusterState(204): Hosts are {cd6be161543e=0} racks are {/default-rack=0} 2024-11-26T20:45:19,403 DEBUG [cd6be161543e:46331 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T20:45:19,403 DEBUG [cd6be161543e:46331 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T20:45:19,403 DEBUG [cd6be161543e:46331 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T20:45:19,403 DEBUG [cd6be161543e:46331 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T20:45:19,403 INFO [cd6be161543e:46331 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T20:45:19,403 INFO [cd6be161543e:46331 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T20:45:19,403 DEBUG [cd6be161543e:46331 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T20:45:19,404 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=cd6be161543e,46687,1732653918727 2024-11-26T20:45:19,405 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cd6be161543e,46687,1732653918727, state=OPENING 2024-11-26T20:45:19,407 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T20:45:19,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:19,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:19,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:19,409 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:45:19,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:19,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:19,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=cd6be161543e,46687,1732653918727}] 2024-11-26T20:45:19,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:19,562 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T20:45:19,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49273, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T20:45:19,567 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T20:45:19,567 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:45:19,567 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T20:45:19,568 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=cd6be161543e%2C46687%2C1732653918727.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,46687,1732653918727, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs, maxLogs=55 2024-11-26T20:45:19,587 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,46687,1732653918727/cd6be161543e%2C46687%2C1732653918727.meta.1732653919569.meta 2024-11-26T20:45:19,587 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T20:45:19,588 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:45:19,588 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T20:45:19,588 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T20:45:19,588 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T20:45:19,588 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T20:45:19,588 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:19,589 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T20:45:19,589 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T20:45:19,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T20:45:19,591 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T20:45:19,591 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:19,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:19,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T20:45:19,592 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T20:45:19,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:19,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:19,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T20:45:19,593 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T20:45:19,593 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:19,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:19,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T20:45:19,594 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T20:45:19,594 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:19,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:45:19,594 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T20:45:19,595 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740 2024-11-26T20:45:19,595 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740 2024-11-26T20:45:19,595 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T20:45:19,595 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T20:45:19,595 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:45:19,596 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T20:45:19,597 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74081729, jitterRate=0.10390378534793854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:45:19,597 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T20:45:19,598 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732653919589Writing region info on filesystem at 1732653919589Initializing all the Stores at 1732653919589Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653919589Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653919590 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653919590Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653919590Cleaning up temporary data from old regions at 1732653919595 (+5 ms)Running coprocessor post-open hooks at 1732653919597 (+2 ms)Region opened successfully at 1732653919598 (+1 ms) 2024-11-26T20:45:19,598 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732653919562 2024-11-26T20:45:19,601 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T20:45:19,601 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T20:45:19,601 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,46687,1732653918727 2024-11-26T20:45:19,602 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cd6be161543e,46687,1732653918727, state=OPEN 2024-11-26T20:45:19,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:45:19,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:45:19,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:45:19,604 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=cd6be161543e,46687,1732653918727 2024-11-26T20:45:19,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:19,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:19,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:45:19,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T20:45:19,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=cd6be161543e,46687,1732653918727 in 196 msec 2024-11-26T20:45:19,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T20:45:19,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-11-26T20:45:19,609 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:45:19,609 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T20:45:19,612 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:45:19,612 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,46687,1732653918727, seqNum=-1] 2024-11-26T20:45:19,613 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:45:19,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39299, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:45:19,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 672 msec 2024-11-26T20:45:19,619 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732653919619, completionTime=-1 2024-11-26T20:45:19,619 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-26T20:45:19,619 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:45:19,621 INFO [master/cd6be161543e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:45:19,621 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732653979621 2024-11-26T20:45:19,622 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732654039621 2024-11-26T20:45:19,622 INFO [master/cd6be161543e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-26T20:45:19,622 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,46331,1732653918641-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,622 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,46331,1732653918641-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,622 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,46331,1732653918641-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,622 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-cd6be161543e:46331, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,622 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,622 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,625 DEBUG [master/cd6be161543e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T20:45:19,627 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.861sec 2024-11-26T20:45:19,627 INFO [master/cd6be161543e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T20:45:19,627 INFO [master/cd6be161543e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T20:45:19,627 INFO [master/cd6be161543e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T20:45:19,627 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T20:45:19,627 INFO [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T20:45:19,627 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,46331,1732653918641-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:45:19,627 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,46331,1732653918641-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T20:45:19,628 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T20:45:19,628 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T20:45:19,628 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,46331,1732653918641-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:19,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac0971d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:45:19,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,46331,-1 for getting cluster id 2024-11-26T20:45:19,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:45:19,649 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dea3817a-6925-49d9-8e0f-4f6639213d6c' 2024-11-26T20:45:19,649 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:45:19,649 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dea3817a-6925-49d9-8e0f-4f6639213d6c" 2024-11-26T20:45:19,649 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2201258b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:45:19,650 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,46331,-1] 2024-11-26T20:45:19,650 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:45:19,650 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:19,652 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:45:19,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b217088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:45:19,653 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:45:19,655 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,46687,1732653918727, seqNum=-1] 2024-11-26T20:45:19,655 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:45:19,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:45:19,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=cd6be161543e,46331,1732653918641 2024-11-26T20:45:19,660 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:45:19,661 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is cd6be161543e,46331,1732653918641 2024-11-26T20:45:19,661 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f24e5c2 2024-11-26T20:45:19,661 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:45:19,663 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:45:19,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46331 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732653919659', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:45:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46331 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732653919659 2024-11-26T20:45:19,666 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732653919659 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T20:45:19,666 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:19,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46331 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732653919659" procId is: 4 2024-11-26T20:45:19,667 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732653919659 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T20:45:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:45:19,679 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732653919659-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b68de948123cc2c643ad10cb079f6ac5, NAME => 'testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732653919659', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 2024-11-26T20:45:19,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732653919659-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:19,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732653919659-pool-0 {}] regionserver.HRegion(1722): Closing b68de948123cc2c643ad10cb079f6ac5, disabling compactions & flushes 2024-11-26T20:45:19,686 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732653919659-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:19,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732653919659-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:19,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732653919659-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. after waiting 0 ms 2024-11-26T20:45:19,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732653919659-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:19,686 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732653919659-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:19,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732653919659-pool-0 {}] regionserver.HRegion(1676): Region close journal for b68de948123cc2c643ad10cb079f6ac5: Waiting for close lock at 1732653919686Disabling compacts and flushes for region at 1732653919686Disabling writes for close at 1732653919686Writing region close event to WAL at 1732653919686Closed at 1732653919686 2024-11-26T20:45:19,687 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732653919659 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T20:45:19,687 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732653919687"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732653919687"}]},"ts":"1732653919687"} 2024-11-26T20:45:19,690 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T20:45:19,690 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732653919659 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T20:45:19,691 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732653919659","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653919690"}]},"ts":"1732653919690"} 2024-11-26T20:45:19,692 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732653919659, state=ENABLING in hbase:meta 2024-11-26T20:45:19,693 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {cd6be161543e=0} racks are {/default-rack=0} 2024-11-26T20:45:19,693 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T20:45:19,693 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T20:45:19,693 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T20:45:19,693 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T20:45:19,693 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T20:45:19,693 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T20:45:19,693 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T20:45:19,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN}] 2024-11-26T20:45:19,694 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN 2024-11-26T20:45:19,695 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN; state=OFFLINE, location=cd6be161543e,37675,1732653918690; forceNewPlan=false, retain=false 2024-11-26T20:45:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:45:19,846 INFO [cd6be161543e:46331 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T20:45:19,846 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b68de948123cc2c643ad10cb079f6ac5, regionState=OPENING, regionLocation=cd6be161543e,37675,1732653918690 2024-11-26T20:45:19,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN because future has completed 2024-11-26T20:45:19,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b68de948123cc2c643ad10cb079f6ac5, server=cd6be161543e,37675,1732653918690}] 2024-11-26T20:45:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:45:20,001 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T20:45:20,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T20:45:20,005 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:20,006 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b68de948123cc2c643ad10cb079f6ac5, NAME => 'testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:45:20,006 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732653919659 b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,006 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:20,006 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,006 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,007 INFO [StoreOpener-b68de948123cc2c643ad10cb079f6ac5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,011 INFO [StoreOpener-b68de948123cc2c643ad10cb079f6ac5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b68de948123cc2c643ad10cb079f6ac5 columnFamilyName family 2024-11-26T20:45:20,011 DEBUG [StoreOpener-b68de948123cc2c643ad10cb079f6ac5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:20,012 INFO [StoreOpener-b68de948123cc2c643ad10cb079f6ac5-1 {}] regionserver.HStore(327): Store=b68de948123cc2c643ad10cb079f6ac5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:20,012 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,013 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,013 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,013 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,013 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,013 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,019 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:45:20,020 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b68de948123cc2c643ad10cb079f6ac5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60505036, jitterRate=-0.09840470552444458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:45:20,020 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:20,020 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b68de948123cc2c643ad10cb079f6ac5: Running coprocessor pre-open hook at 1732653920006Writing region info on filesystem at 1732653920006Initializing all the Stores at 1732653920007 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653920007Cleaning up temporary data from old regions at 1732653920013 (+6 ms)Running coprocessor post-open hooks at 1732653920020 (+7 ms)Region opened successfully at 1732653920020 2024-11-26T20:45:20,021 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5., pid=6, masterSystemTime=1732653920001 2024-11-26T20:45:20,023 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:20,023 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:20,024 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b68de948123cc2c643ad10cb079f6ac5, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,37675,1732653918690 2024-11-26T20:45:20,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b68de948123cc2c643ad10cb079f6ac5, server=cd6be161543e,37675,1732653918690 because future has completed 2024-11-26T20:45:20,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T20:45:20,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b68de948123cc2c643ad10cb079f6ac5, server=cd6be161543e,37675,1732653918690 in 178 msec 2024-11-26T20:45:20,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T20:45:20,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN in 336 msec 2024-11-26T20:45:20,032 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732653919659 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T20:45:20,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732653919659","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653920032"}]},"ts":"1732653920032"} 2024-11-26T20:45:20,034 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732653919659, state=ENABLED in hbase:meta 2024-11-26T20:45:20,034 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732653919659 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T20:45:20,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732653919659 in 370 msec 2024-11-26T20:45:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:45:20,296 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732653919659 completed 2024-11-26T20:45:20,296 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732653919659 get assigned. Timeout = 60000ms 2024-11-26T20:45:20,296 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:45:20,299 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732653919659 assigned to meta. Checking AM states. 2024-11-26T20:45:20,300 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:45:20,300 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732653919659 assigned. 2024-11-26T20:45:20,303 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732653919659', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5., hostname=cd6be161543e,37675,1732653918690, seqNum=2] 2024-11-26T20:45:20,303 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:45:20,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:45:20,309 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b68de948123cc2c643ad10cb079f6ac5 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-26T20:45:20,320 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ff8c6f3e9f394483a45d8f75497557bd 2024-11-26T20:45:20,333 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/.tmp/family/ff8c6f3e9f394483a45d8f75497557bd is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-26T20:45:20,333 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/.tmp/family/ff8c6f3e9f394483a45d8f75497557bd 2024-11-26T20:45:20,336 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/.tmp/family/ff8c6f3e9f394483a45d8f75497557bd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd 2024-11-26T20:45:20,337 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd, delay=938 ms 2024-11-26T20:45:20,338 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd, entries=2, sequenceid=6, filesize=4.9 K 2024-11-26T20:45:20,339 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for b68de948123cc2c643ad10cb079f6ac5 in 30ms, sequenceid=6, compaction requested=false 2024-11-26T20:45:20,339 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732653919659' 2024-11-26T20:45:20,340 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b68de948123cc2c643ad10cb079f6ac5: 2024-11-26T20:45:20,972 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:45:21,276 DEBUG [hfile-prefetch-1732653918077 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ff8c6f3e9f394483a45d8f75497557bd_0. 2024-11-26T20:45:21,277 DEBUG [hfile-prefetch-1732653918077 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ff8c6f3e9f394483a45d8f75497557bd_117. 2024-11-26T20:45:21,277 DEBUG [hfile-prefetch-1732653918077 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd, with total blocks 2, and data blocks 1 2024-11-26T20:45:21,277 DEBUG [hfile-prefetch-1732653918077 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T20:45:21,277 DEBUG [hfile-prefetch-1732653918077 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T20:45:21,277 DEBUG [hfile-prefetch-1732653918077 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd. Will sleep 100ms and try the verification again. 2024-11-26T20:45:21,301 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T20:45:21,301 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T20:45:21,301 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T20:45:21,301 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T20:45:21,305 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T20:45:21,305 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732653921301 2024-11-26T20:45:21,305 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T20:45:21,340 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;cd6be161543e:37675,5,FailOnTimeoutGroup] 2024-11-26T20:45:21,340 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cd6be161543e,37675,1732653918690' ***** 2024-11-26T20:45:21,340 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-26T20:45:21,341 INFO [RS:0;cd6be161543e:37675 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T20:45:21,341 INFO [RS:0;cd6be161543e:37675 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.persistence 2024-11-26T20:45:21,341 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T20:45:21,341 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T20:45:21,341 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T20:45:21,341 INFO [RS:0;cd6be161543e:37675 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T20:45:21,341 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T20:45:21,351 DEBUG [RS:0;cd6be161543e:37675 {}] bucket.BucketCache(1487): Thread RS:0;cd6be161543e:37675 started persisting bucket cache to file 2024-11-26T20:45:21,351 DEBUG [RS:0;cd6be161543e:37675 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T20:45:21,351 DEBUG [RS:0;cd6be161543e:37675 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T20:45:21,355 DEBUG [RS:0;cd6be161543e:37675 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T20:45:21,355 DEBUG [RS:0;cd6be161543e:37675 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732653921351 2024-11-26T20:45:21,355 DEBUG [RS:0;cd6be161543e:37675 {}] bucket.BucketCache(1509): Thread RS:0;cd6be161543e:37675 finished persisting bucket cache to file, renaming 2024-11-26T20:45:21,355 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T20:45:21,355 INFO [RS:0;cd6be161543e:37675 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T20:45:21,355 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T20:45:21,355 INFO [RS:0;cd6be161543e:37675 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T20:45:21,355 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(3091): Received CLOSE for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,356 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(959): stopping server cd6be161543e,37675,1732653918690 2024-11-26T20:45:21,356 INFO [RS:0;cd6be161543e:37675 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:45:21,356 INFO [RS:0;cd6be161543e:37675 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;cd6be161543e:37675. 2024-11-26T20:45:21,356 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T20:45:21,356 DEBUG [RS:0;cd6be161543e:37675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:45:21,356 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b68de948123cc2c643ad10cb079f6ac5, disabling compactions & flushes 2024-11-26T20:45:21,356 DEBUG [RS:0;cd6be161543e:37675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:21,356 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:21,356 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:21,356 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T20:45:21,356 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. after waiting 0 ms 2024-11-26T20:45:21,356 DEBUG [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(1325): Online Regions={b68de948123cc2c643ad10cb079f6ac5=testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5.} 2024-11-26T20:45:21,356 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:21,356 DEBUG [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(1351): Waiting on b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,357 DEBUG [StoreFileCloser-b68de948123cc2c643ad10cb079f6ac5-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd 2024-11-26T20:45:21,357 WARN [hfile-prefetch-1732653918077 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-26T20:45:21,357 DEBUG [hfile-prefetch-1732653918077 {}] hfile.PrefetchExecutor(121): Prefetch completed for ff8c6f3e9f394483a45d8f75497557bd 2024-11-26T20:45:21,361 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T20:45:21,362 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:21,362 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b68de948123cc2c643ad10cb079f6ac5: Waiting for close lock at 1732653921356Running coprocessor pre-close hooks at 1732653921356Disabling compacts and flushes for region at 1732653921356Disabling writes for close at 1732653921356Writing region close event to WAL at 1732653921357 (+1 ms)Running coprocessor post-close hooks at 1732653921361 (+4 ms)Closed at 1732653921361 2024-11-26T20:45:21,362 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:21,399 INFO [regionserver/cd6be161543e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:21,556 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(976): stopping server cd6be161543e,37675,1732653918690; all regions closed. 2024-11-26T20:45:21,558 DEBUG [RS:0;cd6be161543e:37675 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs 2024-11-26T20:45:21,558 INFO [RS:0;cd6be161543e:37675 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cd6be161543e%2C37675%2C1732653918690:(num 1732653919235) 2024-11-26T20:45:21,558 DEBUG [RS:0;cd6be161543e:37675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:21,558 INFO [RS:0;cd6be161543e:37675 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:21,559 INFO [RS:0;cd6be161543e:37675 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:45:21,559 INFO [RS:0;cd6be161543e:37675 {}] hbase.ChoreService(370): Chore service for: regionserver/cd6be161543e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T20:45:21,559 INFO [RS:0;cd6be161543e:37675 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T20:45:21,559 INFO [RS:0;cd6be161543e:37675 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T20:45:21,559 INFO [RS:0;cd6be161543e:37675 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T20:45:21,559 INFO [regionserver/cd6be161543e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:45:21,559 INFO [RS:0;cd6be161543e:37675 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:45:21,559 INFO [RS:0;cd6be161543e:37675 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37675 2024-11-26T20:45:21,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cd6be161543e,37675,1732653918690 2024-11-26T20:45:21,563 INFO [RS:0;cd6be161543e:37675 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:45:21,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:45:21,566 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cd6be161543e,37675,1732653918690] 2024-11-26T20:45:21,567 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cd6be161543e,37675,1732653918690 already deleted, retry=false 2024-11-26T20:45:21,567 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of cd6be161543e,37675,1732653918690 on cd6be161543e,46331,1732653918641 2024-11-26T20:45:21,573 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure cd6be161543e,37675,1732653918690, splitWal=true, meta=false 2024-11-26T20:45:21,576 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure cd6be161543e,37675,1732653918690, splitWal=true, meta=false 2024-11-26T20:45:21,576 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for cd6be161543e,37675,1732653918690 (carryingMeta=false) cd6be161543e,37675,1732653918690/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2aeae813[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T20:45:21,577 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): cd6be161543e,37675,1732653918690 had 1 regions 2024-11-26T20:45:21,577 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure cd6be161543e,37675,1732653918690, splitWal=true, meta=false, isMeta: false 2024-11-26T20:45:21,578 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,37675,1732653918690-splitting 2024-11-26T20:45:21,578 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,37675,1732653918690-splitting dir is empty, no logs to split. 2024-11-26T20:45:21,578 INFO [PEWorker-2 {}] master.SplitWALManager(105): cd6be161543e,37675,1732653918690 WAL count=0, meta=false 2024-11-26T20:45:21,580 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,37675,1732653918690-splitting dir is empty, no logs to split. 2024-11-26T20:45:21,580 INFO [PEWorker-2 {}] master.SplitWALManager(105): cd6be161543e,37675,1732653918690 WAL count=0, meta=false 2024-11-26T20:45:21,580 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if cd6be161543e,37675,1732653918690 WAL splitting is done? wals=0, meta=false 2024-11-26T20:45:21,580 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for cd6be161543e,37675,1732653918690 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,37675,1732653918690-splitting does not exist 2024-11-26T20:45:21,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN}] 2024-11-26T20:45:21,582 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN 2024-11-26T20:45:21,583 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:45:21,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:21,666 INFO [RS:0;cd6be161543e:37675 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:45:21,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37675-0x101560694310001, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:21,666 INFO [RS:0;cd6be161543e:37675 {}] regionserver.HRegionServer(1031): Exiting; stopping=cd6be161543e,37675,1732653918690; zookeeper connection closed. 2024-11-26T20:45:21,666 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@12bc89cf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@12bc89cf 2024-11-26T20:45:21,734 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b68de948123cc2c643ad10cb079f6ac5, regionState=OPENING, regionLocation=cd6be161543e,46687,1732653918727 2024-11-26T20:45:21,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN because future has completed 2024-11-26T20:45:21,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b68de948123cc2c643ad10cb079f6ac5, server=cd6be161543e,46687,1732653918727}] 2024-11-26T20:45:21,857 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cd6be161543e:0 server-side Connection retries=45 2024-11-26T20:45:21,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:21,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:21,857 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:45:21,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:45:21,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:45:21,857 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T20:45:21,858 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:45:21,858 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36189 2024-11-26T20:45:21,860 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36189 connecting to ZooKeeper ensemble=127.0.0.1:54450 2024-11-26T20:45:21,860 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:21,860 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:21,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361890x0, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:45:21,863 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:361890x0, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-26T20:45:21,864 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-26T20:45:21,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36189-0x101560694310003 connected 2024-11-26T20:45:21,864 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T20:45:21,865 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.cache 2024-11-26T20:45:21,865 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T20:45:21,865 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T20:45:21,865 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T20:45:21,868 INFO [Thread-199 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T20:45:21,868 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T20:45:21,868 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T20:45:21,869 INFO [Thread-199 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-26T20:45:21,869 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36189-0x101560694310003, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T20:45:21,870 INFO [Thread-199 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-26T20:45:21,870 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36189-0x101560694310003, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:45:21,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36189 2024-11-26T20:45:21,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36189 2024-11-26T20:45:21,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36189 2024-11-26T20:45:21,873 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36189 2024-11-26T20:45:21,873 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36189 2024-11-26T20:45:21,873 DEBUG [Thread-199 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-26T20:45:21,874 INFO [Thread-199 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T20:45:21,874 INFO [Thread-199 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-26T20:45:21,874 INFO [Thread-199 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.persistence is complete. 2024-11-26T20:45:21,874 INFO [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(746): ClusterId : dea3817a-6925-49d9-8e0f-4f6639213d6c 2024-11-26T20:45:21,875 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T20:45:21,877 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T20:45:21,877 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T20:45:21,880 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T20:45:21,880 DEBUG [RS:2;cd6be161543e:36189 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e90c890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cd6be161543e/172.17.0.2:0 2024-11-26T20:45:21,891 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:21,891 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b68de948123cc2c643ad10cb079f6ac5, NAME => 'testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:45:21,891 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732653919659 b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,891 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:45:21,892 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,892 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,892 INFO [StoreOpener-b68de948123cc2c643ad10cb079f6ac5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,893 DEBUG [RS:2;cd6be161543e:36189 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;cd6be161543e:36189 2024-11-26T20:45:21,893 INFO [RS:2;cd6be161543e:36189 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T20:45:21,893 INFO [RS:2;cd6be161543e:36189 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T20:45:21,893 DEBUG [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T20:45:21,893 INFO [StoreOpener-b68de948123cc2c643ad10cb079f6ac5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b68de948123cc2c643ad10cb079f6ac5 columnFamilyName family 2024-11-26T20:45:21,893 DEBUG [StoreOpener-b68de948123cc2c643ad10cb079f6ac5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:45:21,894 INFO [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(2659): reportForDuty to master=cd6be161543e,46331,1732653918641 with port=36189, startcode=1732653921857 2024-11-26T20:45:21,894 DEBUG [RS:2;cd6be161543e:36189 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T20:45:21,895 DEBUG [StoreFileOpener-b68de948123cc2c643ad10cb079f6ac5-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd, delay=990 ms 2024-11-26T20:45:21,897 DEBUG [StoreOpener-b68de948123cc2c643ad10cb079f6ac5-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd 2024-11-26T20:45:21,897 INFO [StoreOpener-b68de948123cc2c643ad10cb079f6ac5-1 {}] regionserver.HStore(327): Store=b68de948123cc2c643ad10cb079f6ac5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:45:21,897 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,897 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45289, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T20:45:21,897 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cd6be161543e,36189,1732653921857 2024-11-26T20:45:21,898 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46331 {}] master.ServerManager(517): Registering regionserver=cd6be161543e,36189,1732653921857 2024-11-26T20:45:21,898 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,898 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,898 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,898 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,899 DEBUG [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31 2024-11-26T20:45:21,899 DEBUG [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T20:45:21,899 DEBUG [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T20:45:21,900 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,900 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened b68de948123cc2c643ad10cb079f6ac5; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60898217, jitterRate=-0.09254585206508636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:45:21,900 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:21,901 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b68de948123cc2c643ad10cb079f6ac5: Running coprocessor pre-open hook at 1732653921892Writing region info on filesystem at 1732653921892Initializing all the Stores at 1732653921892Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653921892Cleaning up temporary data from old regions at 1732653921898 (+6 ms)Running coprocessor post-open hooks at 1732653921900 (+2 ms)Region opened successfully at 1732653921901 (+1 ms) 2024-11-26T20:45:21,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:45:21,901 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5., pid=9, masterSystemTime=1732653921889 2024-11-26T20:45:21,902 DEBUG [RS:2;cd6be161543e:36189 {}] zookeeper.ZKUtil(111): regionserver:36189-0x101560694310003, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cd6be161543e,36189,1732653921857 2024-11-26T20:45:21,902 WARN [RS:2;cd6be161543e:36189 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:45:21,902 INFO [RS:2;cd6be161543e:36189 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:45:21,902 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cd6be161543e,36189,1732653921857] 2024-11-26T20:45:21,902 DEBUG [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,36189,1732653921857 2024-11-26T20:45:21,904 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:21,904 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:21,904 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b68de948123cc2c643ad10cb079f6ac5, regionState=OPEN, openSeqNum=10, regionLocation=cd6be161543e,46687,1732653918727 2024-11-26T20:45:21,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b68de948123cc2c643ad10cb079f6ac5, server=cd6be161543e,46687,1732653918727 because future has completed 2024-11-26T20:45:21,909 INFO [RS:2;cd6be161543e:36189 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T20:45:21,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-26T20:45:21,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure b68de948123cc2c643ad10cb079f6ac5, server=cd6be161543e,46687,1732653918727 in 171 msec 2024-11-26T20:45:21,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T20:45:21,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732653919659, region=b68de948123cc2c643ad10cb079f6ac5, ASSIGN in 327 msec 2024-11-26T20:45:21,911 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server cd6be161543e,37675,1732653918690 after splitting done 2024-11-26T20:45:21,912 INFO [RS:2;cd6be161543e:36189 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T20:45:21,912 INFO [RS:2;cd6be161543e:36189 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T20:45:21,912 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,913 INFO [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T20:45:21,913 INFO [RS:2;cd6be161543e:36189 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T20:45:21,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure cd6be161543e,37675,1732653918690, splitWal=true, meta=false in 342 msec 2024-11-26T20:45:21,914 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,914 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,914 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,914 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,914 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,914 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,914 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cd6be161543e:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:45:21,914 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,914 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,915 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,915 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,915 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,915 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:45:21,915 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:21,915 DEBUG [RS:2;cd6be161543e:36189 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:45:21,915 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,916 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,916 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,916 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,916 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,916 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,36189,1732653921857-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:45:21,934 INFO [RS:2;cd6be161543e:36189 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T20:45:21,935 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,36189,1732653921857-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,935 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,935 INFO [RS:2;cd6be161543e:36189 {}] regionserver.Replication(171): cd6be161543e,36189,1732653921857 started 2024-11-26T20:45:21,949 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:45:21,949 INFO [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(1482): Serving as cd6be161543e,36189,1732653921857, RpcServer on cd6be161543e/172.17.0.2:36189, sessionid=0x101560694310003 2024-11-26T20:45:21,949 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T20:45:21,949 DEBUG [RS:2;cd6be161543e:36189 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cd6be161543e,36189,1732653921857 2024-11-26T20:45:21,949 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,36189,1732653921857' 2024-11-26T20:45:21,949 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T20:45:21,950 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T20:45:21,950 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T20:45:21,950 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T20:45:21,950 DEBUG [RS:2;cd6be161543e:36189 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cd6be161543e,36189,1732653921857 2024-11-26T20:45:21,950 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,36189,1732653921857' 2024-11-26T20:45:21,951 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T20:45:21,951 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T20:45:21,951 DEBUG [RS:2;cd6be161543e:36189 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T20:45:21,951 INFO [RS:2;cd6be161543e:36189 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T20:45:21,951 INFO [RS:2;cd6be161543e:36189 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T20:45:22,052 INFO [RS:2;cd6be161543e:36189 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:45:22,053 INFO [RS:2;cd6be161543e:36189 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=cd6be161543e%2C36189%2C1732653921857, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,36189,1732653921857, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs, maxLogs=55 2024-11-26T20:45:22,073 INFO [RS:2;cd6be161543e:36189 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/WALs/cd6be161543e,36189,1732653921857/cd6be161543e%2C36189%2C1732653921857.1732653922054 2024-11-26T20:45:22,073 DEBUG [RS:2;cd6be161543e:36189 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T20:45:22,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T20:45:22,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:45:22,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:45:22,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:22,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:22,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:45:22,450 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T20:45:22,450 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=796838939, stopped=false 2024-11-26T20:45:22,450 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=cd6be161543e,46331,1732653918641 2024-11-26T20:45:22,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:22,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:22,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36189-0x101560694310003, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:45:22,452 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:45:22,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:22,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:22,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36189-0x101560694310003, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:22,452 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:45:22,452 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:45:22,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:22,453 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cd6be161543e,46687,1732653918727' ***** 2024-11-26T20:45:22,453 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T20:45:22,453 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cd6be161543e,36189,1732653921857' ***** 2024-11-26T20:45:22,453 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T20:45:22,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36189-0x101560694310003, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:22,453 INFO [RS:1;cd6be161543e:46687 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T20:45:22,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:22,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:45:22,453 INFO [RS:1;cd6be161543e:46687 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.persistence 2024-11-26T20:45:22,453 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T20:45:22,453 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T20:45:22,453 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T20:45:22,453 INFO [RS:1;cd6be161543e:46687 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T20:45:22,453 INFO [RS:2;cd6be161543e:36189 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T20:45:22,453 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T20:45:22,454 DEBUG [RS:1;cd6be161543e:46687 {}] bucket.BucketCache(1487): Thread RS:1;cd6be161543e:46687 started persisting bucket cache to file 2024-11-26T20:45:22,454 DEBUG [RS:1;cd6be161543e:46687 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T20:45:22,454 INFO [RS:2;cd6be161543e:36189 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26bfc0f7-b663-74aa-54ab-953e772a6d1b/bucket.persistence 2024-11-26T20:45:22,454 DEBUG [RS:1;cd6be161543e:46687 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T20:45:22,454 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T20:45:22,454 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T20:45:22,454 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T20:45:22,454 INFO [RS:2;cd6be161543e:36189 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T20:45:22,454 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T20:45:22,457 DEBUG [RS:1;cd6be161543e:46687 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T20:45:22,458 DEBUG [RS:1;cd6be161543e:46687 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732653922454 2024-11-26T20:45:22,458 DEBUG [RS:1;cd6be161543e:46687 {}] bucket.BucketCache(1509): Thread RS:1;cd6be161543e:46687 finished persisting bucket cache to file, renaming 2024-11-26T20:45:22,458 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T20:45:22,458 INFO [RS:1;cd6be161543e:46687 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T20:45:22,458 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T20:45:22,458 INFO [RS:1;cd6be161543e:46687 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T20:45:22,458 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(3091): Received CLOSE for b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:22,458 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(959): stopping server cd6be161543e,46687,1732653918727 2024-11-26T20:45:22,458 INFO [RS:1;cd6be161543e:46687 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:45:22,458 INFO [RS:1;cd6be161543e:46687 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;cd6be161543e:46687. 2024-11-26T20:45:22,458 DEBUG [RS:1;cd6be161543e:46687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:45:22,458 DEBUG [RS:1;cd6be161543e:46687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:22,458 INFO [RS:1;cd6be161543e:46687 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T20:45:22,458 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T20:45:22,459 INFO [RS:1;cd6be161543e:46687 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T20:45:22,459 INFO [RS:1;cd6be161543e:46687 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T20:45:22,459 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b68de948123cc2c643ad10cb079f6ac5, disabling compactions & flushes 2024-11-26T20:45:22,459 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T20:45:22,459 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:22,459 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:22,459 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. after waiting 0 ms 2024-11-26T20:45:22,459 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T20:45:22,459 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:22,459 DEBUG [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, b68de948123cc2c643ad10cb079f6ac5=testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5.} 2024-11-26T20:45:22,459 DEBUG [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b68de948123cc2c643ad10cb079f6ac5 2024-11-26T20:45:22,459 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T20:45:22,459 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T20:45:22,459 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T20:45:22,459 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T20:45:22,459 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T20:45:22,459 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T20:45:22,459 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-26T20:45:22,459 DEBUG [StoreFileCloser-b68de948123cc2c643ad10cb079f6ac5-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/family/ff8c6f3e9f394483a45d8f75497557bd 2024-11-26T20:45:22,464 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/default/testBlockEvictionOnGracefulStop1732653919659/b68de948123cc2c643ad10cb079f6ac5/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-26T20:45:22,464 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:22,464 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b68de948123cc2c643ad10cb079f6ac5: Waiting for close lock at 1732653922458Running coprocessor pre-close hooks at 1732653922459 (+1 ms)Disabling compacts and flushes for region at 1732653922459Disabling writes for close at 1732653922459Writing region close event to WAL at 1732653922460 (+1 ms)Running coprocessor post-close hooks at 1732653922464 (+4 ms)Closed at 1732653922464 2024-11-26T20:45:22,464 DEBUG [RS:2;cd6be161543e:36189 {}] bucket.BucketCache(1487): Thread RS:2;cd6be161543e:36189 started persisting bucket cache to file 2024-11-26T20:45:22,464 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5. 2024-11-26T20:45:22,464 DEBUG [RS:2;cd6be161543e:36189 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T20:45:22,464 DEBUG [RS:2;cd6be161543e:36189 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T20:45:22,468 DEBUG [RS:2;cd6be161543e:36189 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T20:45:22,468 DEBUG [RS:2;cd6be161543e:36189 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732653922464 2024-11-26T20:45:22,468 DEBUG [RS:2;cd6be161543e:36189 {}] bucket.BucketCache(1509): Thread RS:2;cd6be161543e:36189 finished persisting bucket cache to file, renaming 2024-11-26T20:45:22,468 INFO [RS:2;cd6be161543e:36189 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T20:45:22,468 INFO [RS:2;cd6be161543e:36189 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T20:45:22,468 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T20:45:22,468 INFO [RS:2;cd6be161543e:36189 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T20:45:22,468 INFO [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(959): stopping server cd6be161543e,36189,1732653921857 2024-11-26T20:45:22,468 INFO [RS:2;cd6be161543e:36189 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:45:22,468 INFO [RS:2;cd6be161543e:36189 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;cd6be161543e:36189. 2024-11-26T20:45:22,468 DEBUG [RS:2;cd6be161543e:36189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:45:22,468 DEBUG [RS:2;cd6be161543e:36189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:22,468 INFO [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(976): stopping server cd6be161543e,36189,1732653921857; all regions closed. 2024-11-26T20:45:22,469 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7eb6fb6c107c4d04b8d469ac3b40d1e9 2024-11-26T20:45:22,475 DEBUG [RS:2;cd6be161543e:36189 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs 2024-11-26T20:45:22,475 INFO [RS:2;cd6be161543e:36189 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cd6be161543e%2C36189%2C1732653921857:(num 1732653922054) 2024-11-26T20:45:22,475 DEBUG [RS:2;cd6be161543e:36189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:22,475 INFO [RS:2;cd6be161543e:36189 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:22,475 INFO [RS:2;cd6be161543e:36189 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:45:22,475 INFO [RS:2;cd6be161543e:36189 {}] hbase.ChoreService(370): Chore service for: regionserver/cd6be161543e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T20:45:22,475 INFO [RS:2;cd6be161543e:36189 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T20:45:22,475 INFO [RS:2;cd6be161543e:36189 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T20:45:22,475 INFO [regionserver/cd6be161543e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:45:22,475 INFO [RS:2;cd6be161543e:36189 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T20:45:22,476 INFO [RS:2;cd6be161543e:36189 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:45:22,476 INFO [RS:2;cd6be161543e:36189 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36189 2024-11-26T20:45:22,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:45:22,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36189-0x101560694310003, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cd6be161543e,36189,1732653921857 2024-11-26T20:45:22,479 INFO [RS:2;cd6be161543e:36189 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:45:22,480 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cd6be161543e,36189,1732653921857] 2024-11-26T20:45:22,482 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cd6be161543e,36189,1732653921857 already deleted, retry=false 2024-11-26T20:45:22,482 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cd6be161543e,36189,1732653921857 expired; onlineServers=1 2024-11-26T20:45:22,484 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/info/7eb6fb6c107c4d04b8d469ac3b40d1e9 is 209, key is testBlockEvictionOnGracefulStop1732653919659,,1732653919663.b68de948123cc2c643ad10cb079f6ac5./info:regioninfo/1732653921904/Put/seqid=0 2024-11-26T20:45:22,485 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/info/7eb6fb6c107c4d04b8d469ac3b40d1e9 2024-11-26T20:45:22,493 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d067a36d3f9b4ace8d1ef176abe5cce8 2024-11-26T20:45:22,505 INFO [regionserver/cd6be161543e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:22,507 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/ns/d067a36d3f9b4ace8d1ef176abe5cce8 is 43, key is default/ns:d/1732653919615/Put/seqid=0 2024-11-26T20:45:22,507 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/ns/d067a36d3f9b4ace8d1ef176abe5cce8 2024-11-26T20:45:22,516 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2365f334e0a5435f8eb9f095e16acd05 2024-11-26T20:45:22,517 INFO [regionserver/cd6be161543e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:22,529 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/table/2365f334e0a5435f8eb9f095e16acd05 is 80, key is testBlockEvictionOnGracefulStop1732653919659/table:state/1732653920032/Put/seqid=0 2024-11-26T20:45:22,530 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/table/2365f334e0a5435f8eb9f095e16acd05 2024-11-26T20:45:22,532 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/info/7eb6fb6c107c4d04b8d469ac3b40d1e9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/info/7eb6fb6c107c4d04b8d469ac3b40d1e9 2024-11-26T20:45:22,533 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/info/7eb6fb6c107c4d04b8d469ac3b40d1e9, delay=959 ms 2024-11-26T20:45:22,534 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/info/7eb6fb6c107c4d04b8d469ac3b40d1e9, entries=14, sequenceid=13, filesize=7.5 K 2024-11-26T20:45:22,534 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/ns/d067a36d3f9b4ace8d1ef176abe5cce8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/ns/d067a36d3f9b4ace8d1ef176abe5cce8 2024-11-26T20:45:22,535 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/ns/d067a36d3f9b4ace8d1ef176abe5cce8, delay=993 ms 2024-11-26T20:45:22,536 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/ns/d067a36d3f9b4ace8d1ef176abe5cce8, entries=2, sequenceid=13, filesize=5.0 K 2024-11-26T20:45:22,537 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/.tmp/table/2365f334e0a5435f8eb9f095e16acd05 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/table/2365f334e0a5435f8eb9f095e16acd05 2024-11-26T20:45:22,538 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/table/2365f334e0a5435f8eb9f095e16acd05, delay=957 ms 2024-11-26T20:45:22,538 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/table/2365f334e0a5435f8eb9f095e16acd05, entries=2, sequenceid=13, filesize=5.3 K 2024-11-26T20:45:22,539 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=13, compaction requested=false 2024-11-26T20:45:22,540 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/info/7eb6fb6c107c4d04b8d469ac3b40d1e9 2024-11-26T20:45:22,540 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/ns/d067a36d3f9b4ace8d1ef176abe5cce8 2024-11-26T20:45:22,541 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/table/2365f334e0a5435f8eb9f095e16acd05 2024-11-26T20:45:22,545 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-26T20:45:22,545 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T20:45:22,545 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T20:45:22,545 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732653922459Running coprocessor pre-close hooks at 1732653922459Disabling compacts and flushes for region at 1732653922459Disabling writes for close at 1732653922459Obtaining lock to block concurrent updates at 1732653922459Preparing flush snapshotting stores in 1588230740 at 1732653922459Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732653922460 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732653922460Flushing 1588230740/info: creating writer at 1732653922460Flushing 1588230740/info: appending metadata at 1732653922484 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732653922484Flushing 1588230740/ns: creating writer at 1732653922487 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732653922506 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732653922506Flushing 1588230740/table: creating writer at 1732653922509 (+3 ms)Flushing 1588230740/table: appending metadata at 1732653922528 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732653922528Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a0d0cfb: reopening flushed file at 1732653922532 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fcea3a7: reopening flushed file at 1732653922534 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ba43875: reopening flushed file at 1732653922536 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=13, compaction requested=false at 1732653922539 (+3 ms)Writing region close event to WAL at 1732653922541 (+2 ms)Running coprocessor post-close hooks at 1732653922545 (+4 ms)Closed at 1732653922545 2024-11-26T20:45:22,545 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T20:45:22,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36189-0x101560694310003, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:22,581 INFO [RS:2;cd6be161543e:36189 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:45:22,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36189-0x101560694310003, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:22,581 INFO [RS:2;cd6be161543e:36189 {}] regionserver.HRegionServer(1031): Exiting; stopping=cd6be161543e,36189,1732653921857; zookeeper connection closed. 2024-11-26T20:45:22,581 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1640e66a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1640e66a 2024-11-26T20:45:22,659 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(976): stopping server cd6be161543e,46687,1732653918727; all regions closed. 2024-11-26T20:45:22,661 DEBUG [RS:1;cd6be161543e:46687 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs 2024-11-26T20:45:22,661 INFO [RS:1;cd6be161543e:46687 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cd6be161543e%2C46687%2C1732653918727.meta:.meta(num 1732653919569) 2024-11-26T20:45:22,663 DEBUG [RS:1;cd6be161543e:46687 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/oldWALs 2024-11-26T20:45:22,663 INFO [RS:1;cd6be161543e:46687 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cd6be161543e%2C46687%2C1732653918727:(num 1732653919239) 2024-11-26T20:45:22,663 DEBUG [RS:1;cd6be161543e:46687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:45:22,663 INFO [RS:1;cd6be161543e:46687 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:45:22,663 INFO [RS:1;cd6be161543e:46687 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:45:22,663 INFO [RS:1;cd6be161543e:46687 {}] hbase.ChoreService(370): Chore service for: regionserver/cd6be161543e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T20:45:22,663 INFO [RS:1;cd6be161543e:46687 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:45:22,663 INFO [regionserver/cd6be161543e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:45:22,663 INFO [RS:1;cd6be161543e:46687 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46687 2024-11-26T20:45:22,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cd6be161543e,46687,1732653918727 2024-11-26T20:45:22,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:45:22,665 INFO [RS:1;cd6be161543e:46687 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:45:22,668 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cd6be161543e,46687,1732653918727] 2024-11-26T20:45:22,669 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cd6be161543e,46687,1732653918727 already deleted, retry=false 2024-11-26T20:45:22,669 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cd6be161543e,46687,1732653918727 expired; onlineServers=0 2024-11-26T20:45:22,669 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'cd6be161543e,46331,1732653918641' ***** 2024-11-26T20:45:22,669 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T20:45:22,670 INFO [M:0;cd6be161543e:46331 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:45:22,670 INFO [M:0;cd6be161543e:46331 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:45:22,670 DEBUG [M:0;cd6be161543e:46331 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T20:45:22,670 DEBUG [M:0;cd6be161543e:46331 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T20:45:22,670 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T20:45:22,670 DEBUG [master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.small.0-1732653918957 {}] cleaner.HFileCleaner(306): Exit Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.small.0-1732653918957,5,FailOnTimeoutGroup] 2024-11-26T20:45:22,670 DEBUG [master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.large.0-1732653918957 {}] cleaner.HFileCleaner(306): Exit Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.large.0-1732653918957,5,FailOnTimeoutGroup] 2024-11-26T20:45:22,670 INFO [M:0;cd6be161543e:46331 {}] hbase.ChoreService(370): Chore service for: master/cd6be161543e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T20:45:22,670 INFO [M:0;cd6be161543e:46331 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:45:22,670 DEBUG [M:0;cd6be161543e:46331 {}] master.HMaster(1795): Stopping service threads 2024-11-26T20:45:22,670 INFO [M:0;cd6be161543e:46331 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T20:45:22,670 INFO [M:0;cd6be161543e:46331 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:45:22,671 INFO [M:0;cd6be161543e:46331 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:45:22,671 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T20:45:22,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T20:45:22,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:45:22,672 DEBUG [M:0;cd6be161543e:46331 {}] zookeeper.ZKUtil(347): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T20:45:22,672 WARN [M:0;cd6be161543e:46331 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T20:45:22,672 INFO [M:0;cd6be161543e:46331 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/.lastflushedseqids 2024-11-26T20:45:22,679 INFO [M:0;cd6be161543e:46331 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:45:22,679 INFO [M:0;cd6be161543e:46331 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T20:45:22,679 DEBUG [M:0;cd6be161543e:46331 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T20:45:22,679 INFO [M:0;cd6be161543e:46331 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:22,679 DEBUG [M:0;cd6be161543e:46331 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:22,679 DEBUG [M:0;cd6be161543e:46331 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T20:45:22,679 DEBUG [M:0;cd6be161543e:46331 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:22,679 INFO [M:0;cd6be161543e:46331 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-11-26T20:45:22,689 DEBUG [M:0;cd6be161543e:46331 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ccfa7674cd8f4b25968c04127106fa7c 2024-11-26T20:45:22,702 DEBUG [M:0;cd6be161543e:46331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ccfa7674cd8f4b25968c04127106fa7c is 82, key is hbase:meta,,1/info:regioninfo/1732653919601/Put/seqid=0 2024-11-26T20:45:22,703 INFO [M:0;cd6be161543e:46331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ccfa7674cd8f4b25968c04127106fa7c 2024-11-26T20:45:22,710 DEBUG [M:0;cd6be161543e:46331 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on efad760606e44254a6beebcbe8b820f9 2024-11-26T20:45:22,724 DEBUG [M:0;cd6be161543e:46331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/efad760606e44254a6beebcbe8b820f9 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732653920035/Put/seqid=0 2024-11-26T20:45:22,724 INFO [M:0;cd6be161543e:46331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/efad760606e44254a6beebcbe8b820f9 2024-11-26T20:45:22,732 DEBUG [M:0;cd6be161543e:46331 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d475f5c7a9384ecfa8994bb1149a18fe 2024-11-26T20:45:22,746 DEBUG [M:0;cd6be161543e:46331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d475f5c7a9384ecfa8994bb1149a18fe is 69, key is cd6be161543e,36189,1732653921857/rs:state/1732653921898/Put/seqid=0 2024-11-26T20:45:22,746 INFO [M:0;cd6be161543e:46331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d475f5c7a9384ecfa8994bb1149a18fe 2024-11-26T20:45:22,748 INFO [M:0;cd6be161543e:46331 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d475f5c7a9384ecfa8994bb1149a18fe 2024-11-26T20:45:22,748 DEBUG [M:0;cd6be161543e:46331 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ccfa7674cd8f4b25968c04127106fa7c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ccfa7674cd8f4b25968c04127106fa7c 2024-11-26T20:45:22,750 INFO [M:0;cd6be161543e:46331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ccfa7674cd8f4b25968c04127106fa7c, entries=8, sequenceid=83, filesize=5.5 K 2024-11-26T20:45:22,750 DEBUG [M:0;cd6be161543e:46331 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/efad760606e44254a6beebcbe8b820f9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/efad760606e44254a6beebcbe8b820f9 2024-11-26T20:45:22,752 INFO [M:0;cd6be161543e:46331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/efad760606e44254a6beebcbe8b820f9, entries=9, sequenceid=83, filesize=6.4 K 2024-11-26T20:45:22,752 DEBUG [M:0;cd6be161543e:46331 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d475f5c7a9384ecfa8994bb1149a18fe as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d475f5c7a9384ecfa8994bb1149a18fe 2024-11-26T20:45:22,753 INFO [M:0;cd6be161543e:46331 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d475f5c7a9384ecfa8994bb1149a18fe 2024-11-26T20:45:22,753 INFO [M:0;cd6be161543e:46331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1586084d-3f19-9b7d-cf40-4f55394a58b0/c0a78f47-37c6-cfbd-b0a9-211fb9f6de31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d475f5c7a9384ecfa8994bb1149a18fe, entries=3, sequenceid=83, filesize=5.3 K 2024-11-26T20:45:22,754 INFO [M:0;cd6be161543e:46331 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=83, compaction requested=false 2024-11-26T20:45:22,756 INFO [M:0;cd6be161543e:46331 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:45:22,756 DEBUG [M:0;cd6be161543e:46331 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732653922679Disabling compacts and flushes for region at 1732653922679Disabling writes for close at 1732653922679Obtaining lock to block concurrent updates at 1732653922679Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732653922679Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1732653922680 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732653922680Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732653922680Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732653922702 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732653922702Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732653922705 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732653922723 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732653922723Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732653922726 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732653922745 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732653922745Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@426667d9: reopening flushed file at 1732653922748 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3db89302: reopening flushed file at 1732653922750 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15720587: reopening flushed file at 1732653922752 (+2 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=83, compaction requested=false at 1732653922754 (+2 ms)Writing region close event to WAL at 1732653922756 (+2 ms)Closed at 1732653922756 2024-11-26T20:45:22,757 INFO [M:0;cd6be161543e:46331 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T20:45:22,757 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:45:22,757 INFO [M:0;cd6be161543e:46331 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46331 2024-11-26T20:45:22,757 INFO [M:0;cd6be161543e:46331 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:45:22,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:22,768 INFO [RS:1;cd6be161543e:46687 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:45:22,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46687-0x101560694310002, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:22,768 INFO [RS:1;cd6be161543e:46687 {}] regionserver.HRegionServer(1031): Exiting; stopping=cd6be161543e,46687,1732653918727; zookeeper connection closed. 2024-11-26T20:45:22,768 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@33ecbb11 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@33ecbb11 2024-11-26T20:45:22,769 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T20:45:22,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:22,859 INFO [M:0;cd6be161543e:46331 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:45:22,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46331-0x101560694310000, quorum=127.0.0.1:54450, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:45:22,862 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T20:45:22,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T20:45:22,863 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:45:22,870 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 386), ProcessCount=11 (was 11), AvailableMemoryMB=6044 (was 6113)