2024-11-20 08:48:37,190 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 08:48:37,211 main DEBUG Took 0.019235 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 08:48:37,211 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 08:48:37,212 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 08:48:37,213 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 08:48:37,214 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,220 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 08:48:37,232 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,233 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,234 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,234 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,235 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,235 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,236 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,236 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,237 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,237 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,238 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,238 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,238 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,239 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,240 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,240 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,241 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,241 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,241 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,242 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:48:37,243 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,243 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 08:48:37,244 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:48:37,245 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 08:48:37,247 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 08:48:37,248 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 08:48:37,249 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 08:48:37,249 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 08:48:37,258 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 08:48:37,260 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 08:48:37,262 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 08:48:37,262 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 08:48:37,262 main DEBUG createAppenders(={Console}) 2024-11-20 08:48:37,263 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-20 08:48:37,263 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 08:48:37,263 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-20 08:48:37,264 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 08:48:37,264 main DEBUG OutputStream closed 2024-11-20 08:48:37,264 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 08:48:37,265 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 08:48:37,265 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-20 08:48:37,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 08:48:37,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 08:48:37,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 08:48:37,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 08:48:37,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 08:48:37,342 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 08:48:37,342 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 08:48:37,343 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 08:48:37,343 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 08:48:37,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 08:48:37,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 08:48:37,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 08:48:37,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 08:48:37,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 08:48:37,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 08:48:37,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 08:48:37,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 08:48:37,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 08:48:37,351 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 08:48:37,351 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-20 08:48:37,352 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 08:48:37,353 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-20T08:48:37,367 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-20 08:48:37,370 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 08:48:37,370 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T08:48:37,622 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14 2024-11-20T08:48:37,646 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=996, ProcessCount=11, AvailableMemoryMB=3287 2024-11-20T08:48:37,752 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T08:48:37,848 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:37,863 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/cluster_fc825c19-ac22-dfa5-dcc7-b0c8854951bf, deleteOnExit=true 2024-11-20T08:48:37,998 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/cluster_fc825c19-ac22-dfa5-dcc7-b0c8854951bf/zookeeper_0, clientPort=53143, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/cluster_fc825c19-ac22-dfa5-dcc7-b0c8854951bf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/cluster_fc825c19-ac22-dfa5-dcc7-b0c8854951bf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T08:48:38,014 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53143 2024-11-20T08:48:38,014 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:38,100 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 with version=8 2024-11-20T08:48:38,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/hbase-staging 2024-11-20T08:48:38,200 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T08:48:38,527 INFO [Time-limited test {}] client.ConnectionUtils(128): master/be1eb2620e0e:0 server-side Connection retries=45 2024-11-20T08:48:38,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:38,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:38,548 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T08:48:38,549 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:38,549 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T08:48:38,698 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T08:48:38,788 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T08:48:38,829 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T08:48:38,844 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T08:48:38,850 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T08:48:38,893 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21007 (auto-detected) 2024-11-20T08:48:38,894 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T08:48:38,925 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38889 2024-11-20T08:48:38,953 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38889 connecting to ZooKeeper ensemble=127.0.0.1:53143 2024-11-20T08:48:38,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:388890x0, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T08:48:38,993 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38889-0x100141e173c0000 connected 2024-11-20T08:48:39,028 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:39,029 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:39,039 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:39,043 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239, hbase.cluster.distributed=false 2024-11-20T08:48:39,068 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T08:48:39,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38889 2024-11-20T08:48:39,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38889 2024-11-20T08:48:39,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38889 2024-11-20T08:48:39,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38889 2024-11-20T08:48:39,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38889 2024-11-20T08:48:39,164 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/be1eb2620e0e:0 server-side Connection retries=45 2024-11-20T08:48:39,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:39,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:39,168 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T08:48:39,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:39,169 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T08:48:39,172 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T08:48:39,175 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T08:48:39,176 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34937 2024-11-20T08:48:39,178 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34937 connecting to ZooKeeper ensemble=127.0.0.1:53143 2024-11-20T08:48:39,179 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:39,179 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:39,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:349370x0, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T08:48:39,186 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:39,186 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34937-0x100141e173c0001 connected 2024-11-20T08:48:39,192 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T08:48:39,207 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.cache 2024-11-20T08:48:39,215 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T08:48:39,216 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T08:48:39,218 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T08:48:39,220 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T08:48:39,220 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T08:48:39,231 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T08:48:39,232 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T08:48:39,232 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.persistence is complete. 2024-11-20T08:48:39,239 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T08:48:39,243 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T08:48:39,250 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T08:48:39,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34937 2024-11-20T08:48:39,258 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34937 2024-11-20T08:48:39,258 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34937 2024-11-20T08:48:39,260 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34937 2024-11-20T08:48:39,260 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34937 2024-11-20T08:48:39,290 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/be1eb2620e0e:0 server-side Connection retries=45 2024-11-20T08:48:39,290 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:39,291 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:39,291 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T08:48:39,291 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:39,292 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T08:48:39,292 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T08:48:39,292 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T08:48:39,296 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38849 2024-11-20T08:48:39,299 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38849 connecting to ZooKeeper ensemble=127.0.0.1:53143 2024-11-20T08:48:39,300 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:39,301 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:39,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388490x0, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T08:48:39,308 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388490x0, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:39,309 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T08:48:39,312 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38849-0x100141e173c0002 connected 2024-11-20T08:48:39,316 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.cache 2024-11-20T08:48:39,317 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T08:48:39,317 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T08:48:39,317 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T08:48:39,318 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T08:48:39,318 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T08:48:39,318 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T08:48:39,319 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T08:48:39,319 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T08:48:39,319 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.persistence is complete. 2024-11-20T08:48:39,321 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T08:48:39,323 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T08:48:39,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38849 2024-11-20T08:48:39,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38849 2024-11-20T08:48:39,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38849 2024-11-20T08:48:39,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38849 2024-11-20T08:48:39,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38849 2024-11-20T08:48:39,355 DEBUG [M:0;be1eb2620e0e:38889 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;be1eb2620e0e:38889 2024-11-20T08:48:39,358 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:39,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:39,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:39,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:39,372 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:39,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T08:48:39,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:39,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:39,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T08:48:39,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:39,402 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T08:48:39,404 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/be1eb2620e0e,38889,1732092518288 from backup master directory 2024-11-20T08:48:39,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:39,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:39,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:39,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:39,409 WARN [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T08:48:39,410 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:39,413 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T08:48:39,415 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T08:48:39,423 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/hbase.id] with ID: f12522aa-ad26-4169-828e-adbbd14823bf 2024-11-20T08:48:39,423 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/.tmp/hbase.id 2024-11-20T08:48:39,431 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/hbase.id] 2024-11-20T08:48:39,491 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:39,497 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T08:48:39,526 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-20T08:48:39,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:39,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:39,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:39,570 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T08:48:39,575 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T08:48:39,628 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:48:39,635 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:39,708 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store 2024-11-20T08:48:39,732 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T08:48:39,736 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:39,737 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T08:48:39,738 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:39,738 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:39,740 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T08:48:39,740 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:39,740 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:39,741 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732092519737Disabling compacts and flushes for region at 1732092519737Disabling writes for close at 1732092519740 (+3 ms)Writing region close event to WAL at 1732092519740Closed at 1732092519740 2024-11-20T08:48:39,741 WARN [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/.initializing 2024-11-20T08:48:39,742 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/WALs/be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:39,755 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T08:48:39,772 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=be1eb2620e0e%2C38889%2C1732092518288, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/WALs/be1eb2620e0e,38889,1732092518288, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/oldWALs, maxLogs=10 2024-11-20T08:48:39,820 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/WALs/be1eb2620e0e,38889,1732092518288/be1eb2620e0e%2C38889%2C1732092518288.1732092519777 2024-11-20T08:48:39,821 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:39,822 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:39,822 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:39,826 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,827 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,905 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T08:48:39,909 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:39,911 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:39,912 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,918 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T08:48:39,918 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:39,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:39,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T08:48:39,924 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:39,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:39,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T08:48:39,929 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:39,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:39,930 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,935 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,936 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,941 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,941 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,945 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T08:48:39,948 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:39,955 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:48:39,956 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64380451, jitterRate=-0.040656521916389465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T08:48:39,966 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732092519840Initializing all the Stores at 1732092519841 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092519842 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092519844 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092519844Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092519844Cleaning up temporary data from old regions at 1732092519942 (+98 ms)Region opened successfully at 1732092519965 (+23 ms) 2024-11-20T08:48:39,968 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T08:48:40,004 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13aa50d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=be1eb2620e0e/172.17.0.2:0 2024-11-20T08:48:40,044 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T08:48:40,060 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T08:48:40,060 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T08:48:40,064 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T08:48:40,065 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T08:48:40,072 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T08:48:40,072 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T08:48:40,102 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T08:48:40,111 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T08:48:40,114 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T08:48:40,117 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T08:48:40,118 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T08:48:40,120 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T08:48:40,122 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T08:48:40,126 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T08:48:40,128 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T08:48:40,130 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T08:48:40,131 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T08:48:40,150 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T08:48:40,152 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T08:48:40,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:40,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:40,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:40,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,172 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=be1eb2620e0e,38889,1732092518288, sessionid=0x100141e173c0000, setting cluster-up flag (Was=false) 2024-11-20T08:48:40,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,193 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T08:48:40,195 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:40,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,205 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T08:48:40,207 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:40,213 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T08:48:40,236 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(746): ClusterId : f12522aa-ad26-4169-828e-adbbd14823bf 2024-11-20T08:48:40,236 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(746): ClusterId : f12522aa-ad26-4169-828e-adbbd14823bf 2024-11-20T08:48:40,240 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T08:48:40,240 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T08:48:40,245 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T08:48:40,245 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T08:48:40,245 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T08:48:40,245 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T08:48:40,249 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T08:48:40,249 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T08:48:40,249 DEBUG [RS:1;be1eb2620e0e:38849 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be8a142, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=be1eb2620e0e/172.17.0.2:0 2024-11-20T08:48:40,249 DEBUG [RS:0;be1eb2620e0e:34937 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@166ab269, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=be1eb2620e0e/172.17.0.2:0 2024-11-20T08:48:40,265 DEBUG [RS:1;be1eb2620e0e:38849 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;be1eb2620e0e:38849 2024-11-20T08:48:40,268 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T08:48:40,268 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T08:48:40,268 DEBUG [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T08:48:40,269 DEBUG [RS:0;be1eb2620e0e:34937 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;be1eb2620e0e:34937 2024-11-20T08:48:40,269 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T08:48:40,269 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T08:48:40,269 DEBUG [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T08:48:40,271 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(2659): reportForDuty to master=be1eb2620e0e,38889,1732092518288 with port=38849, startcode=1732092519289 2024-11-20T08:48:40,271 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(2659): reportForDuty to master=be1eb2620e0e,38889,1732092518288 with port=34937, startcode=1732092519123 2024-11-20T08:48:40,284 DEBUG [RS:1;be1eb2620e0e:38849 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T08:48:40,284 DEBUG [RS:0;be1eb2620e0e:34937 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T08:48:40,305 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T08:48:40,317 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T08:48:40,325 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T08:48:40,331 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: be1eb2620e0e,38889,1732092518288 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T08:48:40,343 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:48:40,343 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:48:40,343 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:48:40,344 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:48:40,344 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/be1eb2620e0e:0, corePoolSize=10, maxPoolSize=10 2024-11-20T08:48:40,344 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,344 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T08:48:40,345 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,351 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T08:48:40,352 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732092550352 2024-11-20T08:48:40,352 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T08:48:40,354 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T08:48:40,355 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T08:48:40,359 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T08:48:40,359 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T08:48:40,359 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34813, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T08:48:40,359 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T08:48:40,359 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T08:48:40,360 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:40,360 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T08:48:40,360 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T08:48:40,360 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,366 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38889 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:48:40,368 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T08:48:40,369 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T08:48:40,370 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T08:48:40,372 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T08:48:40,373 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T08:48:40,373 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38889 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:48:40,381 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T08:48:40,382 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 2024-11-20T08:48:40,382 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.large.0-1732092520381,5,FailOnTimeoutGroup] 2024-11-20T08:48:40,383 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.small.0-1732092520383,5,FailOnTimeoutGroup] 2024-11-20T08:48:40,383 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,383 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T08:48:40,384 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,385 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,396 DEBUG [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T08:48:40,396 DEBUG [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T08:48:40,396 WARN [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T08:48:40,396 WARN [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T08:48:40,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:40,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T08:48:40,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T08:48:40,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:40,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:40,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T08:48:40,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T08:48:40,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:40,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:40,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T08:48:40,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T08:48:40,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:40,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:40,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T08:48:40,419 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T08:48:40,419 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:40,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:40,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T08:48:40,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:40,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:40,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T08:48:40,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T08:48:40,422 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T08:48:40,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T08:48:40,429 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:48:40,429 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68066641, jitterRate=0.014271989464759827}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T08:48:40,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732092520402Initializing all the Stores at 1732092520402Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092520402Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092520402Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092520402Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092520402Cleaning up temporary data from old regions at 1732092520422 (+20 ms)Region opened successfully at 1732092520432 (+10 ms) 2024-11-20T08:48:40,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T08:48:40,432 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T08:48:40,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T08:48:40,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T08:48:40,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T08:48:40,434 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T08:48:40,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732092520432Disabling compacts and flushes for region at 1732092520432Disabling writes for close at 1732092520433 (+1 ms)Writing region close event to WAL at 1732092520434 (+1 ms)Closed at 1732092520434 2024-11-20T08:48:40,436 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T08:48:40,437 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T08:48:40,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T08:48:40,452 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T08:48:40,454 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T08:48:40,498 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(2659): reportForDuty to master=be1eb2620e0e,38889,1732092518288 with port=34937, startcode=1732092519123 2024-11-20T08:48:40,498 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(2659): reportForDuty to master=be1eb2620e0e,38889,1732092518288 with port=38849, startcode=1732092519289 2024-11-20T08:48:40,500 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38889 {}] master.ServerManager(363): Checking decommissioned status of RegionServer be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:40,503 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38889 {}] master.ServerManager(517): Registering regionserver=be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:40,511 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38889 {}] master.ServerManager(363): Checking decommissioned status of RegionServer be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:40,511 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38889 {}] master.ServerManager(517): Registering regionserver=be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:40,511 DEBUG [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 2024-11-20T08:48:40,511 DEBUG [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T08:48:40,511 DEBUG [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T08:48:40,513 DEBUG [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 2024-11-20T08:48:40,513 DEBUG [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T08:48:40,513 DEBUG [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T08:48:40,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T08:48:40,522 DEBUG [RS:0;be1eb2620e0e:34937 {}] zookeeper.ZKUtil(111): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:40,523 WARN [RS:0;be1eb2620e0e:34937 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T08:48:40,523 INFO [RS:0;be1eb2620e0e:34937 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:40,523 DEBUG [RS:1;be1eb2620e0e:38849 {}] zookeeper.ZKUtil(111): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:40,523 WARN [RS:1;be1eb2620e0e:38849 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T08:48:40,523 DEBUG [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:40,524 INFO [RS:1;be1eb2620e0e:38849 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:40,524 DEBUG [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:40,525 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [be1eb2620e0e,38849,1732092519289] 2024-11-20T08:48:40,525 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [be1eb2620e0e,34937,1732092519123] 2024-11-20T08:48:40,552 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T08:48:40,552 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T08:48:40,567 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T08:48:40,567 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T08:48:40,572 INFO [RS:0;be1eb2620e0e:34937 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T08:48:40,572 INFO [RS:1;be1eb2620e0e:38849 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T08:48:40,573 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,573 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,574 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T08:48:40,574 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T08:48:40,579 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T08:48:40,579 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T08:48:40,580 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,580 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,581 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,581 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/be1eb2620e0e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T08:48:40,581 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/be1eb2620e0e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T08:48:40,582 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,582 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:40,583 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:40,583 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:40,583 DEBUG [RS:0;be1eb2620e0e:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:40,583 DEBUG [RS:1;be1eb2620e0e:38849 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:40,584 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,585 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,585 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,585 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,585 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,585 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,34937,1732092519123-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T08:48:40,588 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,588 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,588 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,588 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,589 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,589 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,38849,1732092519289-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T08:48:40,605 WARN [be1eb2620e0e:38889 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T08:48:40,609 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T08:48:40,611 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,34937,1732092519123-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,611 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T08:48:40,611 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,611 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,38849,1732092519289-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,611 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.Replication(171): be1eb2620e0e,34937,1732092519123 started 2024-11-20T08:48:40,612 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,612 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.Replication(171): be1eb2620e0e,38849,1732092519289 started 2024-11-20T08:48:40,630 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,631 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(1482): Serving as be1eb2620e0e,34937,1732092519123, RpcServer on be1eb2620e0e/172.17.0.2:34937, sessionid=0x100141e173c0001 2024-11-20T08:48:40,631 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:40,632 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(1482): Serving as be1eb2620e0e,38849,1732092519289, RpcServer on be1eb2620e0e/172.17.0.2:38849, sessionid=0x100141e173c0002 2024-11-20T08:48:40,632 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T08:48:40,632 DEBUG [RS:0;be1eb2620e0e:34937 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:40,632 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T08:48:40,632 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,34937,1732092519123' 2024-11-20T08:48:40,632 DEBUG [RS:1;be1eb2620e0e:38849 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:40,632 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T08:48:40,632 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,38849,1732092519289' 2024-11-20T08:48:40,633 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T08:48:40,634 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T08:48:40,634 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T08:48:40,635 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T08:48:40,635 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T08:48:40,635 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T08:48:40,635 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T08:48:40,635 DEBUG [RS:1;be1eb2620e0e:38849 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:40,635 DEBUG [RS:0;be1eb2620e0e:34937 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:40,635 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,38849,1732092519289' 2024-11-20T08:48:40,635 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T08:48:40,635 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,34937,1732092519123' 2024-11-20T08:48:40,635 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T08:48:40,636 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T08:48:40,636 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T08:48:40,637 DEBUG [RS:0;be1eb2620e0e:34937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T08:48:40,637 DEBUG [RS:1;be1eb2620e0e:38849 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T08:48:40,637 INFO [RS:1;be1eb2620e0e:38849 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T08:48:40,637 INFO [RS:0;be1eb2620e0e:34937 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T08:48:40,637 INFO [RS:1;be1eb2620e0e:38849 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T08:48:40,637 INFO [RS:0;be1eb2620e0e:34937 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T08:48:40,743 INFO [RS:1;be1eb2620e0e:38849 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T08:48:40,743 INFO [RS:0;be1eb2620e0e:34937 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T08:48:40,744 INFO [RS:1;be1eb2620e0e:38849 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=be1eb2620e0e%2C38849%2C1732092519289, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,38849,1732092519289, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs, maxLogs=55 2024-11-20T08:48:40,744 INFO [RS:0;be1eb2620e0e:34937 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=be1eb2620e0e%2C34937%2C1732092519123, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,34937,1732092519123, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs, maxLogs=55 2024-11-20T08:48:40,770 INFO [RS:0;be1eb2620e0e:34937 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,34937,1732092519123/be1eb2620e0e%2C34937%2C1732092519123.1732092520750 2024-11-20T08:48:40,771 DEBUG [RS:0;be1eb2620e0e:34937 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:40,771 INFO [RS:1;be1eb2620e0e:38849 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,38849,1732092519289/be1eb2620e0e%2C38849%2C1732092519289.1732092520750 2024-11-20T08:48:40,772 DEBUG [RS:1;be1eb2620e0e:38849 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:40,858 DEBUG [be1eb2620e0e:38889 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T08:48:40,870 DEBUG [be1eb2620e0e:38889 {}] balancer.BalancerClusterState(204): Hosts are {be1eb2620e0e=0} racks are {/default-rack=0} 2024-11-20T08:48:40,873 DEBUG [be1eb2620e0e:38889 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T08:48:40,874 DEBUG [be1eb2620e0e:38889 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T08:48:40,874 DEBUG [be1eb2620e0e:38889 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T08:48:40,874 DEBUG [be1eb2620e0e:38889 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T08:48:40,874 INFO [be1eb2620e0e:38889 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T08:48:40,874 INFO [be1eb2620e0e:38889 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T08:48:40,874 DEBUG [be1eb2620e0e:38889 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T08:48:40,882 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:40,888 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as be1eb2620e0e,38849,1732092519289, state=OPENING 2024-11-20T08:48:40,894 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T08:48:40,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:40,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:40,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:40,898 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:40,899 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T08:48:40,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=be1eb2620e0e,38849,1732092519289}] 2024-11-20T08:48:41,077 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T08:48:41,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52261, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T08:48:41,090 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T08:48:41,091 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:41,092 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T08:48:41,093 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=be1eb2620e0e%2C38849%2C1732092519289.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,38849,1732092519289, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs, maxLogs=55 2024-11-20T08:48:41,115 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,38849,1732092519289/be1eb2620e0e%2C38849%2C1732092519289.meta.1732092521095.meta 2024-11-20T08:48:41,116 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:41,117 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:41,120 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T08:48:41,123 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T08:48:41,128 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T08:48:41,133 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T08:48:41,133 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:41,133 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T08:48:41,134 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T08:48:41,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T08:48:41,136 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T08:48:41,136 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:41,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:41,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T08:48:41,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T08:48:41,138 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:41,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:41,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T08:48:41,139 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T08:48:41,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:41,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:41,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T08:48:41,141 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T08:48:41,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:41,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:41,142 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T08:48:41,143 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:41,144 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:41,144 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T08:48:41,144 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T08:48:41,145 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T08:48:41,146 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T08:48:41,146 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65267770, jitterRate=-0.027434438467025757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T08:48:41,147 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T08:48:41,148 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732092521134Writing region info on filesystem at 1732092521134Initializing all the Stores at 1732092521135 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092521135Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092521135Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092521135Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092521135Cleaning up temporary data from old regions at 1732092521144 (+9 ms)Running coprocessor post-open hooks at 1732092521147 (+3 ms)Region opened successfully at 1732092521148 (+1 ms) 2024-11-20T08:48:41,154 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732092521068 2024-11-20T08:48:41,165 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T08:48:41,165 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T08:48:41,166 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:41,167 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as be1eb2620e0e,38849,1732092519289, state=OPEN 2024-11-20T08:48:41,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:41,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:41,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:41,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:41,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:41,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:41,171 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:41,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T08:48:41,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=be1eb2620e0e,38849,1732092519289 in 270 msec 2024-11-20T08:48:41,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T08:48:41,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 733 msec 2024-11-20T08:48:41,180 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T08:48:41,180 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T08:48:41,202 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:48:41,203 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,38849,1732092519289, seqNum=-1] 2024-11-20T08:48:41,226 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:48:41,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52419, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:48:41,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 999 msec 2024-11-20T08:48:41,258 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732092521258, completionTime=-1 2024-11-20T08:48:41,262 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T08:48:41,262 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T08:48:41,289 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T08:48:41,289 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732092581289 2024-11-20T08:48:41,289 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732092641289 2024-11-20T08:48:41,289 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-20T08:48:41,293 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T08:48:41,300 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,38889,1732092518288-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:41,301 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,38889,1732092518288-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:41,301 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,38889,1732092518288-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:41,303 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-be1eb2620e0e:38889, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:41,303 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:41,303 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:41,311 DEBUG [master/be1eb2620e0e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T08:48:41,337 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.927sec 2024-11-20T08:48:41,339 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T08:48:41,340 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T08:48:41,341 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T08:48:41,341 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T08:48:41,341 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T08:48:41,342 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,38889,1732092518288-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T08:48:41,343 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,38889,1732092518288-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T08:48:41,344 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T08:48:41,345 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T08:48:41,346 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,38889,1732092518288-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:41,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32be4008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:48:41,449 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T08:48:41,449 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T08:48:41,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,38889,-1 for getting cluster id 2024-11-20T08:48:41,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:48:41,465 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f12522aa-ad26-4169-828e-adbbd14823bf' 2024-11-20T08:48:41,468 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:48:41,468 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f12522aa-ad26-4169-828e-adbbd14823bf" 2024-11-20T08:48:41,469 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b87b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:48:41,469 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,38889,-1] 2024-11-20T08:48:41,471 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:48:41,473 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:41,474 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:48:41,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1061b362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:48:41,478 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:48:41,485 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,38849,1732092519289, seqNum=-1] 2024-11-20T08:48:41,485 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:48:41,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:48:41,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:41,526 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T08:48:41,535 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:41,538 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6a8840d6 2024-11-20T08:48:41,539 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T08:48:41,541 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T08:48:41,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732092521521', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T08:48:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732092521521 2024-11-20T08:48:41,558 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732092521521 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T08:48:41,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732092521521" procId is: 4 2024-11-20T08:48:41,560 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:41,562 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732092521521 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T08:48:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:48:41,576 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732092521521-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d841e4a71f9ebd849d805d27afd96594, NAME => 'testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732092521521', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 2024-11-20T08:48:41,584 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732092521521-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:41,584 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732092521521-pool-0 {}] regionserver.HRegion(1722): Closing d841e4a71f9ebd849d805d27afd96594, disabling compactions & flushes 2024-11-20T08:48:41,584 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732092521521-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:41,584 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732092521521-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:41,585 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732092521521-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. after waiting 0 ms 2024-11-20T08:48:41,585 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732092521521-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:41,585 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732092521521-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:41,585 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732092521521-pool-0 {}] regionserver.HRegion(1676): Region close journal for d841e4a71f9ebd849d805d27afd96594: Waiting for close lock at 1732092521584Disabling compacts and flushes for region at 1732092521584Disabling writes for close at 1732092521585 (+1 ms)Writing region close event to WAL at 1732092521585Closed at 1732092521585 2024-11-20T08:48:41,587 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732092521521 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T08:48:41,594 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732092521587"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732092521587"}]},"ts":"1732092521587"} 2024-11-20T08:48:41,599 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T08:48:41,600 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732092521521 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T08:48:41,603 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732092521521","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732092521600"}]},"ts":"1732092521600"} 2024-11-20T08:48:41,607 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732092521521, state=ENABLING in hbase:meta 2024-11-20T08:48:41,608 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {be1eb2620e0e=0} racks are {/default-rack=0} 2024-11-20T08:48:41,609 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T08:48:41,609 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T08:48:41,609 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T08:48:41,609 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T08:48:41,609 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T08:48:41,609 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T08:48:41,609 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T08:48:41,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, ASSIGN}] 2024-11-20T08:48:41,612 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, ASSIGN 2024-11-20T08:48:41,613 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, ASSIGN; state=OFFLINE, location=be1eb2620e0e,38849,1732092519289; forceNewPlan=false, retain=false 2024-11-20T08:48:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:48:41,766 INFO [be1eb2620e0e:38889 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T08:48:41,767 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d841e4a71f9ebd849d805d27afd96594, regionState=OPENING, regionLocation=be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:41,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, ASSIGN because future has completed 2024-11-20T08:48:41,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d841e4a71f9ebd849d805d27afd96594, server=be1eb2620e0e,38849,1732092519289}] 2024-11-20T08:48:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:48:41,929 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:41,929 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d841e4a71f9ebd849d805d27afd96594, NAME => 'testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:41,930 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732092521521 d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,930 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:41,930 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,930 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,931 INFO [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,936 INFO [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d841e4a71f9ebd849d805d27afd96594 columnFamilyName family 2024-11-20T08:48:41,936 DEBUG [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:41,937 INFO [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] regionserver.HStore(327): Store=d841e4a71f9ebd849d805d27afd96594/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:41,937 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,938 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,938 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,939 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,939 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,939 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,947 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:48:41,947 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d841e4a71f9ebd849d805d27afd96594; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73674398, jitterRate=0.09783408045768738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T08:48:41,947 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:41,949 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d841e4a71f9ebd849d805d27afd96594: Running coprocessor pre-open hook at 1732092521930Writing region info on filesystem at 1732092521930Initializing all the Stores at 1732092521931 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092521931Cleaning up temporary data from old regions at 1732092521939 (+8 ms)Running coprocessor post-open hooks at 1732092521947 (+8 ms)Region opened successfully at 1732092521949 (+2 ms) 2024-11-20T08:48:41,950 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594., pid=6, masterSystemTime=1732092521925 2024-11-20T08:48:41,953 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:41,953 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:41,954 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d841e4a71f9ebd849d805d27afd96594, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:41,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d841e4a71f9ebd849d805d27afd96594, server=be1eb2620e0e,38849,1732092519289 because future has completed 2024-11-20T08:48:41,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T08:48:41,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d841e4a71f9ebd849d805d27afd96594, server=be1eb2620e0e,38849,1732092519289 in 187 msec 2024-11-20T08:48:41,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T08:48:41,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, ASSIGN in 351 msec 2024-11-20T08:48:41,964 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732092521521 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T08:48:41,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732092521521","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732092521964"}]},"ts":"1732092521964"} 2024-11-20T08:48:41,967 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732092521521, state=ENABLED in hbase:meta 2024-11-20T08:48:41,969 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732092521521 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T08:48:41,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732092521521 in 418 msec 2024-11-20T08:48:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:48:42,193 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732092521521 completed 2024-11-20T08:48:42,193 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732092521521 get assigned. Timeout = 60000ms 2024-11-20T08:48:42,196 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T08:48:42,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732092521521 assigned to meta. Checking AM states. 2024-11-20T08:48:42,202 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T08:48:42,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732092521521 assigned. 2024-11-20T08:48:42,213 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732092521521', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594., hostname=be1eb2620e0e,38849,1732092519289, seqNum=2] 2024-11-20T08:48:42,226 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d841e4a71f9ebd849d805d27afd96594 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T08:48:42,255 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0ece61f3db5a4d019a5d857e83aefd4d 2024-11-20T08:48:42,294 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/.tmp/family/0ece61f3db5a4d019a5d857e83aefd4d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T08:48:42,299 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/.tmp/family/0ece61f3db5a4d019a5d857e83aefd4d 2024-11-20T08:48:42,336 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/.tmp/family/0ece61f3db5a4d019a5d857e83aefd4d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d 2024-11-20T08:48:42,338 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d, delay=904 ms 2024-11-20T08:48:42,340 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T08:48:42,348 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for d841e4a71f9ebd849d805d27afd96594 in 120ms, sequenceid=6, compaction requested=false 2024-11-20T08:48:42,349 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732092521521' 2024-11-20T08:48:42,352 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d841e4a71f9ebd849d805d27afd96594: 2024-11-20T08:48:43,245 DEBUG [hfile-prefetch-1732092522338 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0ece61f3db5a4d019a5d857e83aefd4d_0. 2024-11-20T08:48:43,250 DEBUG [hfile-prefetch-1732092522338 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0ece61f3db5a4d019a5d857e83aefd4d_117. 2024-11-20T08:48:43,251 DEBUG [hfile-prefetch-1732092522338 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d, with total blocks 2, and data blocks 1 2024-11-20T08:48:43,251 DEBUG [hfile-prefetch-1732092522338 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T08:48:43,252 DEBUG [hfile-prefetch-1732092522338 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T08:48:43,252 DEBUG [hfile-prefetch-1732092522338 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d. Will sleep 100ms and try the verification again. 2024-11-20T08:48:43,321 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T08:48:43,321 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T08:48:43,321 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:43,321 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:43,322 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T08:48:43,322 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T08:48:43,322 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:43,322 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:43,339 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T08:48:43,339 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732092523322 2024-11-20T08:48:43,339 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T08:48:43,342 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T08:48:43,342 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732092523321 2024-11-20T08:48:43,343 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T08:48:43,353 DEBUG [hfile-prefetch-1732092522338 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d, with total blocks 2, and data blocks 1 2024-11-20T08:48:43,353 DEBUG [hfile-prefetch-1732092522338 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-20T08:48:43,353 DEBUG [hfile-prefetch-1732092522338 {}] bucket.BucketCache(2293): found block for file 0ece61f3db5a4d019a5d857e83aefd4d in the backing map. Acquiring read lock for offset 0 2024-11-20T08:48:43,353 DEBUG [hfile-prefetch-1732092522338 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d has now been fully cached. 2024-11-20T08:48:43,353 DEBUG [hfile-prefetch-1732092522338 {}] hfile.PrefetchExecutor(121): Prefetch completed for 0ece61f3db5a4d019a5d857e83aefd4d 2024-11-20T08:48:43,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=d841e4a71f9ebd849d805d27afd96594, source=be1eb2620e0e,38849,1732092519289, destination=be1eb2620e0e,34937,1732092519123, warming up region on be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T08:48:43,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=d841e4a71f9ebd849d805d27afd96594, source=be1eb2620e0e,38849,1732092519289, destination=be1eb2620e0e,34937,1732092519123, running balancer 2024-11-20T08:48:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, REOPEN/MOVE 2024-11-20T08:48:43,366 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, REOPEN/MOVE 2024-11-20T08:48:43,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60329, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T08:48:43,367 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d841e4a71f9ebd849d805d27afd96594, regionState=CLOSING, regionLocation=be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:43,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.HRegion(7855): Warmup {ENCODED => d841e4a71f9ebd849d805d27afd96594, NAME => 'testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:43,369 INFO [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,370 INFO [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d841e4a71f9ebd849d805d27afd96594 columnFamilyName family 2024-11-20T08:48:43,370 DEBUG [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:43,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, REOPEN/MOVE because future has completed 2024-11-20T08:48:43,371 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-20T08:48:43,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure d841e4a71f9ebd849d805d27afd96594, server=be1eb2620e0e,38849,1732092519289}] 2024-11-20T08:48:43,374 DEBUG [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d 2024-11-20T08:48:43,374 INFO [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] regionserver.HStore(327): Store=d841e4a71f9ebd849d805d27afd96594/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.HRegion(1722): Closing d841e4a71f9ebd849d805d27afd96594, disabling compactions & flushes 2024-11-20T08:48:43,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. after waiting 0 ms 2024-11-20T08:48:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34937 {}] regionserver.HRegion(1676): Region close journal for d841e4a71f9ebd849d805d27afd96594: Waiting for close lock at 1732092523374Disabling compacts and flushes for region at 1732092523374Disabling writes for close at 1732092523374Writing region close event to WAL at 1732092523377 (+3 ms)Closed at 1732092523377 2024-11-20T08:48:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38889 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T08:48:43,530 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,530 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-20T08:48:43,531 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing d841e4a71f9ebd849d805d27afd96594, disabling compactions & flushes 2024-11-20T08:48:43,531 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,531 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,531 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. after waiting 0 ms 2024-11-20T08:48:43,531 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,533 DEBUG [StoreFileCloser-d841e4a71f9ebd849d805d27afd96594-family-1 {}] bucket.BucketCache(768): Removing file 0ece61f3db5a4d019a5d857e83aefd4d for region d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,533 DEBUG [StoreFileCloser-d841e4a71f9ebd849d805d27afd96594-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 0ece61f3db5a4d019a5d857e83aefd4d, starting offset: 0, end offset: 9223372036854775807 2024-11-20T08:48:43,534 DEBUG [StoreFileCloser-d841e4a71f9ebd849d805d27afd96594-family-1 {}] bucket.BucketCache(856): removed key 0ece61f3db5a4d019a5d857e83aefd4d_0 from back map with offset lock 2094080 in the evict process 2024-11-20T08:48:43,539 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T08:48:43,539 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T08:48:43,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:43,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:43,540 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T08:48:43,541 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,541 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for d841e4a71f9ebd849d805d27afd96594: Waiting for close lock at 1732092523531Running coprocessor pre-close hooks at 1732092523531Disabling compacts and flushes for region at 1732092523531Disabling writes for close at 1732092523531Writing region close event to WAL at 1732092523535 (+4 ms)Running coprocessor post-close hooks at 1732092523540 (+5 ms)Closed at 1732092523541 (+1 ms) 2024-11-20T08:48:43,542 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding d841e4a71f9ebd849d805d27afd96594 move to be1eb2620e0e,34937,1732092519123 record at close sequenceid=6 2024-11-20T08:48:43,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T08:48:43,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732092523539 2024-11-20T08:48:43,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T08:48:43,543 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T08:48:43,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T08:48:43,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:43,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:43,545 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,545 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d841e4a71f9ebd849d805d27afd96594, regionState=CLOSED 2024-11-20T08:48:43,546 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T08:48:43,546 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732092523543 2024-11-20T08:48:43,546 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T08:48:43,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure d841e4a71f9ebd849d805d27afd96594, server=be1eb2620e0e,38849,1732092519289 because future has completed 2024-11-20T08:48:43,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T08:48:43,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure d841e4a71f9ebd849d805d27afd96594, server=be1eb2620e0e,38849,1732092519289 in 178 msec 2024-11-20T08:48:43,551 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, REOPEN/MOVE; state=CLOSED, location=be1eb2620e0e,34937,1732092519123; forceNewPlan=false, retain=false 2024-11-20T08:48:43,702 INFO [be1eb2620e0e:38889 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T08:48:43,702 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d841e4a71f9ebd849d805d27afd96594, regionState=OPENING, regionLocation=be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:43,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, REOPEN/MOVE because future has completed 2024-11-20T08:48:43,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d841e4a71f9ebd849d805d27afd96594, server=be1eb2620e0e,34937,1732092519123}] 2024-11-20T08:48:43,862 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,862 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => d841e4a71f9ebd849d805d27afd96594, NAME => 'testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:43,862 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732092521521 d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,862 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:43,863 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,863 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,864 INFO [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,865 INFO [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d841e4a71f9ebd849d805d27afd96594 columnFamilyName family 2024-11-20T08:48:43,865 DEBUG [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:43,866 DEBUG [StoreFileOpener-d841e4a71f9ebd849d805d27afd96594-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d, delay=970 ms 2024-11-20T08:48:43,868 DEBUG [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d 2024-11-20T08:48:43,868 INFO [StoreOpener-d841e4a71f9ebd849d805d27afd96594-1 {}] regionserver.HStore(327): Store=d841e4a71f9ebd849d805d27afd96594/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:43,868 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,869 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,870 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,870 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,870 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,875 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,876 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened d841e4a71f9ebd849d805d27afd96594; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71527126, jitterRate=0.06583723425865173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T08:48:43,876 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,877 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for d841e4a71f9ebd849d805d27afd96594: Running coprocessor pre-open hook at 1732092523863Writing region info on filesystem at 1732092523863Initializing all the Stores at 1732092523863Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092523863Cleaning up temporary data from old regions at 1732092523870 (+7 ms)Running coprocessor post-open hooks at 1732092523876 (+6 ms)Region opened successfully at 1732092523877 (+1 ms) 2024-11-20T08:48:43,878 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594., pid=9, masterSystemTime=1732092523858 2024-11-20T08:48:43,880 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,880 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,881 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d841e4a71f9ebd849d805d27afd96594, regionState=OPEN, openSeqNum=10, regionLocation=be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:43,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d841e4a71f9ebd849d805d27afd96594, server=be1eb2620e0e,34937,1732092519123 because future has completed 2024-11-20T08:48:43,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T08:48:43,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure d841e4a71f9ebd849d805d27afd96594, server=be1eb2620e0e,34937,1732092519123 in 183 msec 2024-11-20T08:48:43,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732092521521, region=d841e4a71f9ebd849d805d27afd96594, REOPEN/MOVE in 526 msec 2024-11-20T08:48:43,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T08:48:43,918 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T08:48:43,918 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:48:43,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:43,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:43,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:48:43,924 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T08:48:43,924 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1596482003, stopped=false 2024-11-20T08:48:43,924 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=be1eb2620e0e,38889,1732092518288 2024-11-20T08:48:43,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:43,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:43,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:43,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:43,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:43,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T08:48:43,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:43,927 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T08:48:43,927 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:48:43,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:43,928 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:43,928 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:43,928 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:43,929 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'be1eb2620e0e,34937,1732092519123' ***** 2024-11-20T08:48:43,929 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T08:48:43,929 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'be1eb2620e0e,38849,1732092519289' ***** 2024-11-20T08:48:43,929 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T08:48:43,929 INFO [RS:0;be1eb2620e0e:34937 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T08:48:43,929 INFO [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T08:48:43,929 INFO [RS:0;be1eb2620e0e:34937 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.persistence 2024-11-20T08:48:43,929 INFO [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/bucket.persistence 2024-11-20T08:48:43,930 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T08:48:43,930 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T08:48:43,930 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T08:48:43,930 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T08:48:43,930 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T08:48:43,930 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T08:48:43,930 INFO [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T08:48:43,930 INFO [RS:0;be1eb2620e0e:34937 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T08:48:43,930 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T08:48:43,930 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T08:48:43,940 DEBUG [RS:0;be1eb2620e0e:34937 {}] bucket.BucketCache(1487): Thread RS:0;be1eb2620e0e:34937 started persisting bucket cache to file 2024-11-20T08:48:43,940 DEBUG [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1487): Thread RS:1;be1eb2620e0e:38849 started persisting bucket cache to file 2024-11-20T08:48:43,940 DEBUG [RS:0;be1eb2620e0e:34937 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:43,940 DEBUG [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:43,940 DEBUG [RS:0;be1eb2620e0e:34937 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:43,940 DEBUG [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:43,944 DEBUG [RS:0;be1eb2620e0e:34937 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T08:48:43,944 DEBUG [RS:0;be1eb2620e0e:34937 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732092523940 2024-11-20T08:48:43,944 DEBUG [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T08:48:43,944 DEBUG [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732092523940 2024-11-20T08:48:43,944 DEBUG [RS:0;be1eb2620e0e:34937 {}] bucket.BucketCache(1509): Thread RS:0;be1eb2620e0e:34937 finished persisting bucket cache to file, renaming 2024-11-20T08:48:43,944 DEBUG [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1509): Thread RS:1;be1eb2620e0e:38849 finished persisting bucket cache to file, renaming 2024-11-20T08:48:43,945 WARN [RS:1;be1eb2620e0e:38849 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-20T08:48:43,945 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T08:48:43,945 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T08:48:43,945 INFO [RS:1;be1eb2620e0e:38849 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T08:48:43,945 INFO [RS:0;be1eb2620e0e:34937 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T08:48:43,945 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T08:48:43,945 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T08:48:43,945 INFO [RS:1;be1eb2620e0e:38849 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T08:48:43,945 INFO [RS:0;be1eb2620e0e:34937 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T08:48:43,945 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(959): stopping server be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:43,945 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(3091): Received CLOSE for d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,945 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T08:48:43,945 INFO [RS:1;be1eb2620e0e:38849 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;be1eb2620e0e:38849. 2024-11-20T08:48:43,946 DEBUG [RS:1;be1eb2620e0e:38849 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:48:43,946 DEBUG [RS:1;be1eb2620e0e:38849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:43,946 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(959): stopping server be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:43,946 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T08:48:43,946 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T08:48:43,946 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T08:48:43,946 INFO [RS:0;be1eb2620e0e:34937 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;be1eb2620e0e:34937. 2024-11-20T08:48:43,946 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T08:48:43,946 DEBUG [RS:0;be1eb2620e0e:34937 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:48:43,946 DEBUG [RS:0;be1eb2620e0e:34937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:43,946 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T08:48:43,946 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T08:48:43,946 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T08:48:43,947 DEBUG [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(1325): Online Regions={d841e4a71f9ebd849d805d27afd96594=testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594.} 2024-11-20T08:48:43,947 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d841e4a71f9ebd849d805d27afd96594, disabling compactions & flushes 2024-11-20T08:48:43,947 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T08:48:43,947 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,947 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,947 DEBUG [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T08:48:43,947 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. after waiting 0 ms 2024-11-20T08:48:43,947 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T08:48:43,947 DEBUG [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(1351): Waiting on d841e4a71f9ebd849d805d27afd96594 2024-11-20T08:48:43,947 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,947 DEBUG [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T08:48:43,947 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T08:48:43,947 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T08:48:43,947 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T08:48:43,947 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T08:48:43,947 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T08:48:43,947 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-20T08:48:43,948 DEBUG [StoreFileCloser-d841e4a71f9ebd849d805d27afd96594-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/family/0ece61f3db5a4d019a5d857e83aefd4d 2024-11-20T08:48:43,953 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnRegionMove1732092521521/d841e4a71f9ebd849d805d27afd96594/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T08:48:43,953 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,953 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d841e4a71f9ebd849d805d27afd96594: Waiting for close lock at 1732092523946Running coprocessor pre-close hooks at 1732092523947 (+1 ms)Disabling compacts and flushes for region at 1732092523947Disabling writes for close at 1732092523947Writing region close event to WAL at 1732092523948 (+1 ms)Running coprocessor post-close hooks at 1732092523953 (+5 ms)Closed at 1732092523953 2024-11-20T08:48:43,953 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594. 2024-11-20T08:48:43,958 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3bb6d23a31ff40918a168ac4d1e36372 2024-11-20T08:48:43,976 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/info/3bb6d23a31ff40918a168ac4d1e36372 is 205, key is testBlockEvictionOnRegionMove1732092521521,,1732092521543.d841e4a71f9ebd849d805d27afd96594./info:regioninfo/1732092523881/Put/seqid=0 2024-11-20T08:48:43,977 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/info/3bb6d23a31ff40918a168ac4d1e36372 2024-11-20T08:48:43,987 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0730f71ff63c46ffbe13402fc1c5d6da 2024-11-20T08:48:43,990 INFO [regionserver/be1eb2620e0e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:43,992 INFO [regionserver/be1eb2620e0e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:44,000 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/ns/0730f71ff63c46ffbe13402fc1c5d6da is 43, key is default/ns:d/1732092521234/Put/seqid=0 2024-11-20T08:48:44,001 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/ns/0730f71ff63c46ffbe13402fc1c5d6da 2024-11-20T08:48:44,010 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on edbfc650ea5c44fab2c9c22d942c4678 2024-11-20T08:48:44,024 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/table/edbfc650ea5c44fab2c9c22d942c4678 is 78, key is testBlockEvictionOnRegionMove1732092521521/table:state/1732092521964/Put/seqid=0 2024-11-20T08:48:44,025 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/table/edbfc650ea5c44fab2c9c22d942c4678 2024-11-20T08:48:44,028 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/info/3bb6d23a31ff40918a168ac4d1e36372 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/3bb6d23a31ff40918a168ac4d1e36372 2024-11-20T08:48:44,029 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/3bb6d23a31ff40918a168ac4d1e36372, delay=927 ms 2024-11-20T08:48:44,030 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/3bb6d23a31ff40918a168ac4d1e36372, entries=15, sequenceid=15, filesize=7.6 K 2024-11-20T08:48:44,031 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/ns/0730f71ff63c46ffbe13402fc1c5d6da as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/0730f71ff63c46ffbe13402fc1c5d6da 2024-11-20T08:48:44,032 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/0730f71ff63c46ffbe13402fc1c5d6da, delay=901 ms 2024-11-20T08:48:44,033 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/0730f71ff63c46ffbe13402fc1c5d6da, entries=2, sequenceid=15, filesize=5.0 K 2024-11-20T08:48:44,033 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/table/edbfc650ea5c44fab2c9c22d942c4678 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/edbfc650ea5c44fab2c9c22d942c4678 2024-11-20T08:48:44,035 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/edbfc650ea5c44fab2c9c22d942c4678, delay=930 ms 2024-11-20T08:48:44,036 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/edbfc650ea5c44fab2c9c22d942c4678, entries=2, sequenceid=15, filesize=5.3 K 2024-11-20T08:48:44,038 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false 2024-11-20T08:48:44,038 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T08:48:44,039 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/3bb6d23a31ff40918a168ac4d1e36372 2024-11-20T08:48:44,040 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/0730f71ff63c46ffbe13402fc1c5d6da 2024-11-20T08:48:44,042 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/edbfc650ea5c44fab2c9c22d942c4678 2024-11-20T08:48:44,047 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T08:48:44,047 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T08:48:44,048 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T08:48:44,048 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732092523947Running coprocessor pre-close hooks at 1732092523947Disabling compacts and flushes for region at 1732092523947Disabling writes for close at 1732092523947Obtaining lock to block concurrent updates at 1732092523947Preparing flush snapshotting stores in 1588230740 at 1732092523947Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732092523948 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732092523948Flushing 1588230740/info: creating writer at 1732092523949 (+1 ms)Flushing 1588230740/info: appending metadata at 1732092523975 (+26 ms)Flushing 1588230740/info: closing flushed file at 1732092523975Flushing 1588230740/ns: creating writer at 1732092523979 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732092524000 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732092524000Flushing 1588230740/table: creating writer at 1732092524004 (+4 ms)Flushing 1588230740/table: appending metadata at 1732092524024 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732092524024Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70054486: reopening flushed file at 1732092524027 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b2effbe: reopening flushed file at 1732092524030 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6edcc010: reopening flushed file at 1732092524033 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false at 1732092524038 (+5 ms)Writing region close event to WAL at 1732092524042 (+4 ms)Running coprocessor post-close hooks at 1732092524047 (+5 ms)Closed at 1732092524048 (+1 ms) 2024-11-20T08:48:44,048 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T08:48:44,147 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(976): stopping server be1eb2620e0e,38849,1732092519289; all regions closed. 2024-11-20T08:48:44,147 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(976): stopping server be1eb2620e0e,34937,1732092519123; all regions closed. 2024-11-20T08:48:44,153 DEBUG [RS:0;be1eb2620e0e:34937 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs 2024-11-20T08:48:44,153 DEBUG [RS:1;be1eb2620e0e:38849 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs 2024-11-20T08:48:44,154 INFO [RS:0;be1eb2620e0e:34937 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C34937%2C1732092519123:(num 1732092520750) 2024-11-20T08:48:44,154 INFO [RS:1;be1eb2620e0e:38849 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C38849%2C1732092519289.meta:.meta(num 1732092521095) 2024-11-20T08:48:44,154 DEBUG [RS:0;be1eb2620e0e:34937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:44,154 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:44,154 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T08:48:44,154 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.ChoreService(370): Chore service for: regionserver/be1eb2620e0e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T08:48:44,154 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T08:48:44,155 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T08:48:44,155 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T08:48:44,155 INFO [regionserver/be1eb2620e0e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T08:48:44,155 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T08:48:44,155 INFO [RS:0;be1eb2620e0e:34937 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34937 2024-11-20T08:48:44,156 DEBUG [RS:1;be1eb2620e0e:38849 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs 2024-11-20T08:48:44,156 INFO [RS:1;be1eb2620e0e:38849 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C38849%2C1732092519289:(num 1732092520750) 2024-11-20T08:48:44,156 DEBUG [RS:1;be1eb2620e0e:38849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:44,156 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:44,156 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T08:48:44,157 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.ChoreService(370): Chore service for: regionserver/be1eb2620e0e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T08:48:44,157 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T08:48:44,157 INFO [regionserver/be1eb2620e0e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T08:48:44,157 INFO [RS:1;be1eb2620e0e:38849 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38849 2024-11-20T08:48:44,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/be1eb2620e0e,34937,1732092519123 2024-11-20T08:48:44,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T08:48:44,161 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T08:48:44,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/be1eb2620e0e,38849,1732092519289 2024-11-20T08:48:44,162 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T08:48:44,164 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [be1eb2620e0e,38849,1732092519289] 2024-11-20T08:48:44,166 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/be1eb2620e0e,38849,1732092519289 already deleted, retry=false 2024-11-20T08:48:44,166 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; be1eb2620e0e,38849,1732092519289 expired; onlineServers=1 2024-11-20T08:48:44,166 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [be1eb2620e0e,34937,1732092519123] 2024-11-20T08:48:44,167 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/be1eb2620e0e,34937,1732092519123 already deleted, retry=false 2024-11-20T08:48:44,167 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; be1eb2620e0e,34937,1732092519123 expired; onlineServers=0 2024-11-20T08:48:44,168 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'be1eb2620e0e,38889,1732092518288' ***** 2024-11-20T08:48:44,168 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T08:48:44,168 INFO [M:0;be1eb2620e0e:38889 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T08:48:44,168 INFO [M:0;be1eb2620e0e:38889 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T08:48:44,168 DEBUG [M:0;be1eb2620e0e:38889 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T08:48:44,168 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T08:48:44,168 DEBUG [M:0;be1eb2620e0e:38889 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T08:48:44,168 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.large.0-1732092520381 {}] cleaner.HFileCleaner(306): Exit Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.large.0-1732092520381,5,FailOnTimeoutGroup] 2024-11-20T08:48:44,168 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.small.0-1732092520383 {}] cleaner.HFileCleaner(306): Exit Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.small.0-1732092520383,5,FailOnTimeoutGroup] 2024-11-20T08:48:44,169 INFO [M:0;be1eb2620e0e:38889 {}] hbase.ChoreService(370): Chore service for: master/be1eb2620e0e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T08:48:44,169 INFO [M:0;be1eb2620e0e:38889 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T08:48:44,169 DEBUG [M:0;be1eb2620e0e:38889 {}] master.HMaster(1795): Stopping service threads 2024-11-20T08:48:44,169 INFO [M:0;be1eb2620e0e:38889 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T08:48:44,169 INFO [M:0;be1eb2620e0e:38889 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T08:48:44,170 INFO [M:0;be1eb2620e0e:38889 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T08:48:44,170 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T08:48:44,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T08:48:44,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,174 DEBUG [M:0;be1eb2620e0e:38889 {}] zookeeper.ZKUtil(347): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T08:48:44,174 WARN [M:0;be1eb2620e0e:38889 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T08:48:44,175 INFO [M:0;be1eb2620e0e:38889 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/.lastflushedseqids 2024-11-20T08:48:44,184 INFO [M:0;be1eb2620e0e:38889 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T08:48:44,185 INFO [M:0;be1eb2620e0e:38889 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T08:48:44,185 DEBUG [M:0;be1eb2620e0e:38889 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T08:48:44,185 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:44,185 DEBUG [M:0;be1eb2620e0e:38889 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:44,185 DEBUG [M:0;be1eb2620e0e:38889 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T08:48:44,185 DEBUG [M:0;be1eb2620e0e:38889 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:44,186 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-20T08:48:44,204 DEBUG [M:0;be1eb2620e0e:38889 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f02885e7b5874eeb97652f9f9b0d9cd6 2024-11-20T08:48:44,219 DEBUG [M:0;be1eb2620e0e:38889 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f02885e7b5874eeb97652f9f9b0d9cd6 is 82, key is hbase:meta,,1/info:regioninfo/1732092521166/Put/seqid=0 2024-11-20T08:48:44,220 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f02885e7b5874eeb97652f9f9b0d9cd6 2024-11-20T08:48:44,229 DEBUG [M:0;be1eb2620e0e:38889 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 973213fb3c82422c99af76a17449457d 2024-11-20T08:48:44,243 DEBUG [M:0;be1eb2620e0e:38889 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/973213fb3c82422c99af76a17449457d is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732092521970/Put/seqid=0 2024-11-20T08:48:44,244 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/973213fb3c82422c99af76a17449457d 2024-11-20T08:48:44,254 DEBUG [M:0;be1eb2620e0e:38889 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8453d049bb914e10b6cdcd507ef43c9b 2024-11-20T08:48:44,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:44,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:44,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38849-0x100141e173c0002, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:44,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x100141e173c0001, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:44,265 INFO [RS:0;be1eb2620e0e:34937 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T08:48:44,265 INFO [RS:1;be1eb2620e0e:38849 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T08:48:44,265 INFO [RS:0;be1eb2620e0e:34937 {}] regionserver.HRegionServer(1031): Exiting; stopping=be1eb2620e0e,34937,1732092519123; zookeeper connection closed. 2024-11-20T08:48:44,265 INFO [RS:1;be1eb2620e0e:38849 {}] regionserver.HRegionServer(1031): Exiting; stopping=be1eb2620e0e,38849,1732092519289; zookeeper connection closed. 2024-11-20T08:48:44,265 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@954f5ab {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@954f5ab 2024-11-20T08:48:44,265 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@12aaccd2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@12aaccd2 2024-11-20T08:48:44,266 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-20T08:48:44,268 DEBUG [M:0;be1eb2620e0e:38889 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8453d049bb914e10b6cdcd507ef43c9b is 69, key is be1eb2620e0e,34937,1732092519123/rs:state/1732092520511/Put/seqid=0 2024-11-20T08:48:44,269 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8453d049bb914e10b6cdcd507ef43c9b 2024-11-20T08:48:44,272 DEBUG [M:0;be1eb2620e0e:38889 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f02885e7b5874eeb97652f9f9b0d9cd6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f02885e7b5874eeb97652f9f9b0d9cd6 2024-11-20T08:48:44,275 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f02885e7b5874eeb97652f9f9b0d9cd6, entries=8, sequenceid=84, filesize=5.5 K 2024-11-20T08:48:44,275 DEBUG [M:0;be1eb2620e0e:38889 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/973213fb3c82422c99af76a17449457d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/973213fb3c82422c99af76a17449457d 2024-11-20T08:48:44,278 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/973213fb3c82422c99af76a17449457d, entries=9, sequenceid=84, filesize=6.5 K 2024-11-20T08:48:44,278 DEBUG [M:0;be1eb2620e0e:38889 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8453d049bb914e10b6cdcd507ef43c9b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8453d049bb914e10b6cdcd507ef43c9b 2024-11-20T08:48:44,280 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8453d049bb914e10b6cdcd507ef43c9b, entries=2, sequenceid=84, filesize=5.1 K 2024-11-20T08:48:44,282 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=84, compaction requested=false 2024-11-20T08:48:44,286 INFO [M:0;be1eb2620e0e:38889 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:44,287 DEBUG [M:0;be1eb2620e0e:38889 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732092524185Disabling compacts and flushes for region at 1732092524185Disabling writes for close at 1732092524185Obtaining lock to block concurrent updates at 1732092524186 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732092524186Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732092524186Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732092524187 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732092524187Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732092524219 (+32 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732092524219Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732092524222 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732092524243 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732092524243Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732092524247 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732092524267 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732092524268 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b2ba34e: reopening flushed file at 1732092524271 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@540ae3e6: reopening flushed file at 1732092524275 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bab2dad: reopening flushed file at 1732092524278 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=84, compaction requested=false at 1732092524282 (+4 ms)Writing region close event to WAL at 1732092524286 (+4 ms)Closed at 1732092524286 2024-11-20T08:48:44,289 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T08:48:44,289 INFO [M:0;be1eb2620e0e:38889 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T08:48:44,289 INFO [M:0;be1eb2620e0e:38889 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38889 2024-11-20T08:48:44,289 INFO [M:0;be1eb2620e0e:38889 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T08:48:44,392 INFO [M:0;be1eb2620e0e:38889 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T08:48:44,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:44,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38889-0x100141e173c0000, quorum=127.0.0.1:53143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:44,396 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T08:48:44,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T08:48:44,426 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:44,435 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/be1eb2620e0e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1732092522338 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732092524029 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/be1eb2620e0e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1732092523867 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/be1eb2620e0e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732092524032 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/be1eb2620e0e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=924 (was 996), ProcessCount=11 (was 11), AvailableMemoryMB=2926 (was 3287) 2024-11-20T08:48:44,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=924, ProcessCount=11, AvailableMemoryMB=2925 2024-11-20T08:48:44,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/hadoop-log-dir so I do NOT create it in target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5 2024-11-20T08:48:44,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/hadoop-tmp-dir so I do NOT create it in target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5 2024-11-20T08:48:44,442 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:44,445 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/cluster_a99de1e0-943e-8ed9-c5a5-d8c4e08c5757, deleteOnExit=true 2024-11-20T08:48:44,448 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/cluster_a99de1e0-943e-8ed9-c5a5-d8c4e08c5757/zookeeper_0, clientPort=62105, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/cluster_a99de1e0-943e-8ed9-c5a5-d8c4e08c5757/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/cluster_a99de1e0-943e-8ed9-c5a5-d8c4e08c5757/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T08:48:44,449 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62105 2024-11-20T08:48:44,463 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 with version=8 2024-11-20T08:48:44,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/hbase-staging 2024-11-20T08:48:44,466 INFO [Time-limited test {}] client.ConnectionUtils(128): master/be1eb2620e0e:0 server-side Connection retries=45 2024-11-20T08:48:44,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:44,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:44,466 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T08:48:44,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:44,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T08:48:44,466 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T08:48:44,467 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T08:48:44,468 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40345 2024-11-20T08:48:44,469 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40345 connecting to ZooKeeper ensemble=127.0.0.1:62105 2024-11-20T08:48:44,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:403450x0, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T08:48:44,476 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40345-0x100141e30850000 connected 2024-11-20T08:48:44,495 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:44,495 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:44,496 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:44,497 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239, hbase.cluster.distributed=false 2024-11-20T08:48:44,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T08:48:44,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40345 2024-11-20T08:48:44,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40345 2024-11-20T08:48:44,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40345 2024-11-20T08:48:44,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40345 2024-11-20T08:48:44,501 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40345 2024-11-20T08:48:44,519 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/be1eb2620e0e:0 server-side Connection retries=45 2024-11-20T08:48:44,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:44,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:44,520 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T08:48:44,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:44,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T08:48:44,520 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T08:48:44,520 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T08:48:44,521 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43559 2024-11-20T08:48:44,523 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43559 connecting to ZooKeeper ensemble=127.0.0.1:62105 2024-11-20T08:48:44,524 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:44,524 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:44,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:435590x0, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T08:48:44,528 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43559-0x100141e30850001 connected 2024-11-20T08:48:44,528 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:44,528 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T08:48:44,529 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.cache 2024-11-20T08:48:44,529 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T08:48:44,530 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T08:48:44,530 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T08:48:44,530 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T08:48:44,530 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T08:48:44,530 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T08:48:44,531 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T08:48:44,531 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.persistence is complete. 2024-11-20T08:48:44,532 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T08:48:44,532 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T08:48:44,534 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T08:48:44,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43559 2024-11-20T08:48:44,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43559 2024-11-20T08:48:44,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43559 2024-11-20T08:48:44,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43559 2024-11-20T08:48:44,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43559 2024-11-20T08:48:44,555 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/be1eb2620e0e:0 server-side Connection retries=45 2024-11-20T08:48:44,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:44,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:44,555 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T08:48:44,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:44,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T08:48:44,555 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T08:48:44,556 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T08:48:44,556 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45375 2024-11-20T08:48:44,558 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45375 connecting to ZooKeeper ensemble=127.0.0.1:62105 2024-11-20T08:48:44,558 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:44,558 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:44,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453750x0, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T08:48:44,562 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453750x0, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:44,562 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45375-0x100141e30850002 connected 2024-11-20T08:48:44,562 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T08:48:44,563 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.cache 2024-11-20T08:48:44,563 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T08:48:44,563 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T08:48:44,563 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T08:48:44,564 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T08:48:44,564 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T08:48:44,564 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T08:48:44,564 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T08:48:44,565 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.persistence is complete. 2024-11-20T08:48:44,565 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T08:48:44,566 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T08:48:44,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T08:48:44,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45375 2024-11-20T08:48:44,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45375 2024-11-20T08:48:44,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45375 2024-11-20T08:48:44,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45375 2024-11-20T08:48:44,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45375 2024-11-20T08:48:44,588 DEBUG [M:0;be1eb2620e0e:40345 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;be1eb2620e0e:40345 2024-11-20T08:48:44,589 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:44,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:44,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:44,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:44,594 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:44,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T08:48:44,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T08:48:44,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,597 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T08:48:44,598 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/be1eb2620e0e,40345,1732092524465 from backup master directory 2024-11-20T08:48:44,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:44,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:44,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:44,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:48:44,600 WARN [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T08:48:44,600 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:44,602 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/hbase.id] with ID: 2798e9c4-7db4-421b-ae34-4225e7304b5d 2024-11-20T08:48:44,602 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/.tmp/hbase.id 2024-11-20T08:48:44,612 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/hbase.id] 2024-11-20T08:48:44,660 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:44,661 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T08:48:44,662 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-20T08:48:44,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,671 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T08:48:44,672 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T08:48:44,672 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:44,698 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store 2024-11-20T08:48:44,705 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:44,705 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T08:48:44,706 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:44,706 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:44,706 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T08:48:44,706 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:44,706 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:44,706 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732092524705Disabling compacts and flushes for region at 1732092524705Disabling writes for close at 1732092524706 (+1 ms)Writing region close event to WAL at 1732092524706Closed at 1732092524706 2024-11-20T08:48:44,706 WARN [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/.initializing 2024-11-20T08:48:44,706 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/WALs/be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:44,713 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T08:48:44,717 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=be1eb2620e0e%2C40345%2C1732092524465, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/WALs/be1eb2620e0e,40345,1732092524465, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/oldWALs, maxLogs=10 2024-11-20T08:48:44,736 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/WALs/be1eb2620e0e,40345,1732092524465/be1eb2620e0e%2C40345%2C1732092524465.1732092524718 2024-11-20T08:48:44,736 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:44,736 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:44,737 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:44,737 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,737 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,738 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,742 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T08:48:44,743 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:44,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:44,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T08:48:44,747 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:44,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:44,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T08:48:44,751 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:44,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:44,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T08:48:44,755 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:44,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:44,755 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,756 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,756 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,756 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,756 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,756 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T08:48:44,757 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:48:44,764 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:48:44,764 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65013392, jitterRate=-0.031224966049194336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T08:48:44,766 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732092524737Initializing all the Stores at 1732092524738 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092524738Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092524738Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092524738Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092524738Cleaning up temporary data from old regions at 1732092524756 (+18 ms)Region opened successfully at 1732092524765 (+9 ms) 2024-11-20T08:48:44,766 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T08:48:44,771 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c0bed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=be1eb2620e0e/172.17.0.2:0 2024-11-20T08:48:44,773 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T08:48:44,773 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T08:48:44,773 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T08:48:44,773 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T08:48:44,774 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T08:48:44,774 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-20T08:48:44,774 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T08:48:44,776 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T08:48:44,778 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T08:48:44,779 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T08:48:44,780 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T08:48:44,781 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T08:48:44,782 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T08:48:44,783 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T08:48:44,784 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T08:48:44,785 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T08:48:44,786 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T08:48:44,788 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T08:48:44,790 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T08:48:44,791 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T08:48:44,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:44,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:44,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:44,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,795 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=be1eb2620e0e,40345,1732092524465, sessionid=0x100141e30850000, setting cluster-up flag (Was=false) 2024-11-20T08:48:44,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,804 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T08:48:44,805 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:44,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:44,816 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T08:48:44,818 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:44,819 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T08:48:44,820 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T08:48:44,820 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T08:48:44,821 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T08:48:44,821 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: be1eb2620e0e,40345,1732092524465 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T08:48:44,823 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:48:44,823 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:48:44,823 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:48:44,823 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:48:44,823 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/be1eb2620e0e:0, corePoolSize=10, maxPoolSize=10 2024-11-20T08:48:44,824 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,824 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T08:48:44,824 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,831 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732092554831 2024-11-20T08:48:44,831 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T08:48:44,831 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T08:48:44,831 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T08:48:44,831 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T08:48:44,831 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T08:48:44,832 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T08:48:44,832 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T08:48:44,832 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T08:48:44,832 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,832 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:44,832 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T08:48:44,833 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T08:48:44,833 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T08:48:44,832 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T08:48:44,834 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T08:48:44,834 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T08:48:44,836 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.large.0-1732092524834,5,FailOnTimeoutGroup] 2024-11-20T08:48:44,840 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.small.0-1732092524836,5,FailOnTimeoutGroup] 2024-11-20T08:48:44,840 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,840 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T08:48:44,840 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,840 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,850 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T08:48:44,850 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 2024-11-20T08:48:44,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:44,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T08:48:44,861 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T08:48:44,861 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:44,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:44,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T08:48:44,866 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T08:48:44,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:44,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:44,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T08:48:44,870 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T08:48:44,870 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:44,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:44,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T08:48:44,875 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T08:48:44,875 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:44,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:44,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T08:48:44,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:44,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:44,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T08:48:44,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T08:48:44,877 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T08:48:44,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T08:48:44,879 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(746): ClusterId : 2798e9c4-7db4-421b-ae34-4225e7304b5d 2024-11-20T08:48:44,879 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(746): ClusterId : 2798e9c4-7db4-421b-ae34-4225e7304b5d 2024-11-20T08:48:44,879 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T08:48:44,879 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T08:48:44,883 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T08:48:44,883 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T08:48:44,883 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T08:48:44,883 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T08:48:44,885 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:48:44,886 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58729532, jitterRate=-0.12486177682876587}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T08:48:44,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732092524857Initializing all the Stores at 1732092524857Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092524857Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092524857Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092524857Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092524857Cleaning up temporary data from old regions at 1732092524877 (+20 ms)Region opened successfully at 1732092524887 (+10 ms) 2024-11-20T08:48:44,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T08:48:44,887 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T08:48:44,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T08:48:44,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T08:48:44,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T08:48:44,888 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T08:48:44,888 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T08:48:44,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732092524887Disabling compacts and flushes for region at 1732092524887Disabling writes for close at 1732092524888 (+1 ms)Writing region close event to WAL at 1732092524888Closed at 1732092524888 2024-11-20T08:48:44,888 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T08:48:44,889 DEBUG [RS:1;be1eb2620e0e:45375 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b7021b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=be1eb2620e0e/172.17.0.2:0 2024-11-20T08:48:44,889 DEBUG [RS:0;be1eb2620e0e:43559 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586fdbe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=be1eb2620e0e/172.17.0.2:0 2024-11-20T08:48:44,891 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T08:48:44,891 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T08:48:44,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T08:48:44,893 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T08:48:44,894 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T08:48:44,908 DEBUG [RS:1;be1eb2620e0e:45375 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;be1eb2620e0e:45375 2024-11-20T08:48:44,908 DEBUG [RS:0;be1eb2620e0e:43559 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;be1eb2620e0e:43559 2024-11-20T08:48:44,909 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T08:48:44,909 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T08:48:44,909 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T08:48:44,909 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T08:48:44,909 DEBUG [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T08:48:44,909 DEBUG [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T08:48:44,910 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(2659): reportForDuty to master=be1eb2620e0e,40345,1732092524465 with port=43559, startcode=1732092524519 2024-11-20T08:48:44,910 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(2659): reportForDuty to master=be1eb2620e0e,40345,1732092524465 with port=45375, startcode=1732092524555 2024-11-20T08:48:44,910 DEBUG [RS:1;be1eb2620e0e:45375 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T08:48:44,910 DEBUG [RS:0;be1eb2620e0e:43559 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T08:48:44,913 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35205, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T08:48:44,913 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41809, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T08:48:44,914 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40345 {}] master.ServerManager(363): Checking decommissioned status of RegionServer be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:44,914 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40345 {}] master.ServerManager(517): Registering regionserver=be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:44,915 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40345 {}] master.ServerManager(363): Checking decommissioned status of RegionServer be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:44,915 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40345 {}] master.ServerManager(517): Registering regionserver=be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:44,915 DEBUG [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 2024-11-20T08:48:44,916 DEBUG [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T08:48:44,916 DEBUG [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T08:48:44,917 DEBUG [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 2024-11-20T08:48:44,917 DEBUG [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T08:48:44,917 DEBUG [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T08:48:44,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T08:48:44,923 DEBUG [RS:0;be1eb2620e0e:43559 {}] zookeeper.ZKUtil(111): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:44,923 WARN [RS:0;be1eb2620e0e:43559 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T08:48:44,923 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [be1eb2620e0e,45375,1732092524555] 2024-11-20T08:48:44,923 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [be1eb2620e0e,43559,1732092524519] 2024-11-20T08:48:44,923 INFO [RS:0;be1eb2620e0e:43559 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:44,923 DEBUG [RS:1;be1eb2620e0e:45375 {}] zookeeper.ZKUtil(111): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:44,923 DEBUG [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:44,923 WARN [RS:1;be1eb2620e0e:45375 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T08:48:44,924 INFO [RS:1;be1eb2620e0e:45375 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:44,924 DEBUG [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:44,929 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T08:48:44,929 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T08:48:44,932 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T08:48:44,933 INFO [RS:1;be1eb2620e0e:45375 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T08:48:44,933 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,936 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T08:48:44,936 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T08:48:44,936 INFO [RS:0;be1eb2620e0e:43559 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T08:48:44,936 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T08:48:44,937 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,937 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,937 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T08:48:44,937 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,937 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,937 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,937 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,937 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,937 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/be1eb2620e0e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T08:48:44,937 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T08:48:44,937 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,938 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:44,938 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 DEBUG [RS:1;be1eb2620e0e:45375 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:44,938 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,938 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,939 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/be1eb2620e0e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T08:48:44,939 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,939 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,939 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,939 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,939 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,939 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:44,939 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:44,940 DEBUG [RS:0;be1eb2620e0e:43559 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:44,941 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,45375,1732092524555-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,941 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,942 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,43559,1732092524519-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T08:48:44,957 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T08:48:44,957 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,45375,1732092524555-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,957 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,958 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.Replication(171): be1eb2620e0e,45375,1732092524555 started 2024-11-20T08:48:44,961 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T08:48:44,961 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,43559,1732092524519-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,961 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,961 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.Replication(171): be1eb2620e0e,43559,1732092524519 started 2024-11-20T08:48:44,972 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,973 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(1482): Serving as be1eb2620e0e,45375,1732092524555, RpcServer on be1eb2620e0e/172.17.0.2:45375, sessionid=0x100141e30850002 2024-11-20T08:48:44,973 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T08:48:44,973 DEBUG [RS:1;be1eb2620e0e:45375 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:44,973 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,45375,1732092524555' 2024-11-20T08:48:44,973 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T08:48:44,974 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T08:48:44,974 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T08:48:44,974 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T08:48:44,974 DEBUG [RS:1;be1eb2620e0e:45375 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:44,974 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,45375,1732092524555' 2024-11-20T08:48:44,974 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T08:48:44,975 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T08:48:44,975 DEBUG [RS:1;be1eb2620e0e:45375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T08:48:44,975 INFO [RS:1;be1eb2620e0e:45375 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T08:48:44,975 INFO [RS:1;be1eb2620e0e:45375 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T08:48:44,978 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:44,979 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(1482): Serving as be1eb2620e0e,43559,1732092524519, RpcServer on be1eb2620e0e/172.17.0.2:43559, sessionid=0x100141e30850001 2024-11-20T08:48:44,979 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T08:48:44,979 DEBUG [RS:0;be1eb2620e0e:43559 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:44,979 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,43559,1732092524519' 2024-11-20T08:48:44,979 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T08:48:44,980 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T08:48:44,980 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T08:48:44,980 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T08:48:44,980 DEBUG [RS:0;be1eb2620e0e:43559 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:44,980 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,43559,1732092524519' 2024-11-20T08:48:44,981 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T08:48:44,981 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T08:48:44,982 DEBUG [RS:0;be1eb2620e0e:43559 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T08:48:44,982 INFO [RS:0;be1eb2620e0e:43559 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T08:48:44,982 INFO [RS:0;be1eb2620e0e:43559 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T08:48:45,044 WARN [be1eb2620e0e:40345 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T08:48:45,076 INFO [RS:1;be1eb2620e0e:45375 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T08:48:45,077 INFO [RS:1;be1eb2620e0e:45375 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=be1eb2620e0e%2C45375%2C1732092524555, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs, maxLogs=55 2024-11-20T08:48:45,083 INFO [RS:0;be1eb2620e0e:43559 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T08:48:45,084 INFO [RS:0;be1eb2620e0e:43559 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=be1eb2620e0e%2C43559%2C1732092524519, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,43559,1732092524519, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs, maxLogs=55 2024-11-20T08:48:45,099 INFO [RS:1;be1eb2620e0e:45375 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555/be1eb2620e0e%2C45375%2C1732092524555.1732092525079 2024-11-20T08:48:45,100 DEBUG [RS:1;be1eb2620e0e:45375 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:45,106 INFO [RS:0;be1eb2620e0e:43559 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,43559,1732092524519/be1eb2620e0e%2C43559%2C1732092524519.1732092525085 2024-11-20T08:48:45,106 DEBUG [RS:0;be1eb2620e0e:43559 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:45,294 DEBUG [be1eb2620e0e:40345 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T08:48:45,295 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(204): Hosts are {be1eb2620e0e=0} racks are {/default-rack=0} 2024-11-20T08:48:45,296 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T08:48:45,296 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T08:48:45,296 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T08:48:45,296 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T08:48:45,296 INFO [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T08:48:45,296 INFO [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T08:48:45,296 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T08:48:45,296 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:45,297 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as be1eb2620e0e,45375,1732092524555, state=OPENING 2024-11-20T08:48:45,299 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T08:48:45,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:45,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:45,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:45,302 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T08:48:45,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=be1eb2620e0e,45375,1732092524555}] 2024-11-20T08:48:45,302 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:45,302 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:45,302 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:45,455 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T08:48:45,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48059, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T08:48:45,460 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T08:48:45,460 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:45,461 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T08:48:45,461 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=be1eb2620e0e%2C45375%2C1732092524555.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs, maxLogs=55 2024-11-20T08:48:45,482 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555/be1eb2620e0e%2C45375%2C1732092524555.meta.1732092525462.meta 2024-11-20T08:48:45,482 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:45,482 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:45,483 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T08:48:45,483 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T08:48:45,483 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T08:48:45,483 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T08:48:45,483 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:45,484 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T08:48:45,484 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T08:48:45,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T08:48:45,486 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T08:48:45,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:45,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:45,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T08:48:45,487 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T08:48:45,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:45,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:45,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T08:48:45,488 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T08:48:45,488 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:45,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:45,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T08:48:45,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T08:48:45,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:45,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:45,489 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T08:48:45,490 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:45,490 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:45,491 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T08:48:45,491 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T08:48:45,491 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T08:48:45,492 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T08:48:45,492 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63429971, jitterRate=-0.054819777607917786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T08:48:45,492 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T08:48:45,493 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732092525484Writing region info on filesystem at 1732092525484Initializing all the Stores at 1732092525484Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092525484Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092525485 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092525486 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092525486Cleaning up temporary data from old regions at 1732092525491 (+5 ms)Running coprocessor post-open hooks at 1732092525493 (+2 ms)Region opened successfully at 1732092525493 2024-11-20T08:48:45,494 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732092525455 2024-11-20T08:48:45,496 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T08:48:45,497 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T08:48:45,497 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:45,498 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as be1eb2620e0e,45375,1732092524555, state=OPEN 2024-11-20T08:48:45,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:45,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:45,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:45,500 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:45,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:45,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:45,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:45,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T08:48:45,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=be1eb2620e0e,45375,1732092524555 in 198 msec 2024-11-20T08:48:45,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T08:48:45,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 610 msec 2024-11-20T08:48:45,504 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T08:48:45,504 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T08:48:45,508 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:48:45,508 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,45375,1732092524555, seqNum=-1] 2024-11-20T08:48:45,508 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:48:45,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:48:45,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 694 msec 2024-11-20T08:48:45,515 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732092525515, completionTime=-1 2024-11-20T08:48:45,515 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T08:48:45,515 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T08:48:45,518 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T08:48:45,518 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732092585518 2024-11-20T08:48:45,518 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732092645518 2024-11-20T08:48:45,518 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-20T08:48:45,518 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T08:48:45,519 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,40345,1732092524465-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:45,519 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,40345,1732092524465-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:45,519 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,40345,1732092524465-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:45,519 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-be1eb2620e0e:40345, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:45,519 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:45,519 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:45,522 DEBUG [master/be1eb2620e0e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T08:48:45,524 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.924sec 2024-11-20T08:48:45,524 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T08:48:45,524 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T08:48:45,524 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T08:48:45,524 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T08:48:45,524 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T08:48:45,524 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,40345,1732092524465-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T08:48:45,525 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,40345,1732092524465-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T08:48:45,525 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T08:48:45,525 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T08:48:45,525 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,40345,1732092524465-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:45,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:48:45,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,40345,-1 for getting cluster id 2024-11-20T08:48:45,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:48:45,581 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2798e9c4-7db4-421b-ae34-4225e7304b5d' 2024-11-20T08:48:45,582 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:48:45,582 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2798e9c4-7db4-421b-ae34-4225e7304b5d" 2024-11-20T08:48:45,582 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594b4991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:48:45,582 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,40345,-1] 2024-11-20T08:48:45,582 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:48:45,583 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:45,584 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:48:45,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:48:45,586 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:48:45,587 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,45375,1732092524555, seqNum=-1] 2024-11-20T08:48:45,587 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:48:45,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:48:45,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:45,592 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T08:48:45,593 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:45,593 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40fc62d7 2024-11-20T08:48:45,593 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T08:48:45,594 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T08:48:45,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732092525591', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T08:48:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732092525591 2024-11-20T08:48:45,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732092525591 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T08:48:45,598 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:45,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732092525591" procId is: 4 2024-11-20T08:48:45,599 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732092525591 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T08:48:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:48:45,610 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732092525591-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7798a5357e73d077d13b95bd29adb960, NAME => 'testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732092525591', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 2024-11-20T08:48:45,619 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732092525591-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:45,619 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732092525591-pool-0 {}] regionserver.HRegion(1722): Closing 7798a5357e73d077d13b95bd29adb960, disabling compactions & flushes 2024-11-20T08:48:45,619 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732092525591-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:45,619 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732092525591-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:45,619 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732092525591-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. after waiting 0 ms 2024-11-20T08:48:45,619 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732092525591-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:45,619 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732092525591-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:45,619 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732092525591-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7798a5357e73d077d13b95bd29adb960: Waiting for close lock at 1732092525619Disabling compacts and flushes for region at 1732092525619Disabling writes for close at 1732092525619Writing region close event to WAL at 1732092525619Closed at 1732092525619 2024-11-20T08:48:45,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732092525591 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T08:48:45,621 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732092525621"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732092525621"}]},"ts":"1732092525621"} 2024-11-20T08:48:45,623 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T08:48:45,624 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732092525591 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T08:48:45,624 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732092525591","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732092525624"}]},"ts":"1732092525624"} 2024-11-20T08:48:45,626 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732092525591, state=ENABLING in hbase:meta 2024-11-20T08:48:45,626 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {be1eb2620e0e=0} racks are {/default-rack=0} 2024-11-20T08:48:45,627 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T08:48:45,627 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T08:48:45,627 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T08:48:45,627 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T08:48:45,627 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T08:48:45,627 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T08:48:45,627 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T08:48:45,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN}] 2024-11-20T08:48:45,628 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN 2024-11-20T08:48:45,629 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN; state=OFFLINE, location=be1eb2620e0e,45375,1732092524555; forceNewPlan=false, retain=false 2024-11-20T08:48:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:48:45,779 INFO [be1eb2620e0e:40345 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T08:48:45,780 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7798a5357e73d077d13b95bd29adb960, regionState=OPENING, regionLocation=be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:45,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN because future has completed 2024-11-20T08:48:45,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7798a5357e73d077d13b95bd29adb960, server=be1eb2620e0e,45375,1732092524555}] 2024-11-20T08:48:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:48:45,938 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:45,938 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7798a5357e73d077d13b95bd29adb960, NAME => 'testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:45,938 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732092525591 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,939 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:45,939 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,939 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,939 INFO [StoreOpener-7798a5357e73d077d13b95bd29adb960-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,944 INFO [StoreOpener-7798a5357e73d077d13b95bd29adb960-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7798a5357e73d077d13b95bd29adb960 columnFamilyName family 2024-11-20T08:48:45,944 DEBUG [StoreOpener-7798a5357e73d077d13b95bd29adb960-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:45,944 INFO [StoreOpener-7798a5357e73d077d13b95bd29adb960-1 {}] regionserver.HStore(327): Store=7798a5357e73d077d13b95bd29adb960/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:45,944 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,945 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,945 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,945 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,945 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,946 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,953 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:48:45,953 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7798a5357e73d077d13b95bd29adb960; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59153633, jitterRate=-0.11854217946529388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T08:48:45,953 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:45,954 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7798a5357e73d077d13b95bd29adb960: Running coprocessor pre-open hook at 1732092525939Writing region info on filesystem at 1732092525939Initializing all the Stores at 1732092525939Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092525939Cleaning up temporary data from old regions at 1732092525945 (+6 ms)Running coprocessor post-open hooks at 1732092525953 (+8 ms)Region opened successfully at 1732092525954 (+1 ms) 2024-11-20T08:48:45,954 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960., pid=6, masterSystemTime=1732092525935 2024-11-20T08:48:45,958 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:45,958 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:45,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7798a5357e73d077d13b95bd29adb960, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:45,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7798a5357e73d077d13b95bd29adb960, server=be1eb2620e0e,45375,1732092524555 because future has completed 2024-11-20T08:48:45,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T08:48:45,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7798a5357e73d077d13b95bd29adb960, server=be1eb2620e0e,45375,1732092524555 in 181 msec 2024-11-20T08:48:45,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T08:48:45,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN in 339 msec 2024-11-20T08:48:45,969 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732092525591 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T08:48:45,969 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732092525591","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732092525969"}]},"ts":"1732092525969"} 2024-11-20T08:48:45,972 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732092525591, state=ENABLED in hbase:meta 2024-11-20T08:48:45,973 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732092525591 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T08:48:45,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732092525591 in 378 msec 2024-11-20T08:48:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:48:46,232 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732092525591 completed 2024-11-20T08:48:46,232 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732092525591 get assigned. Timeout = 60000ms 2024-11-20T08:48:46,233 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T08:48:46,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732092525591 assigned to meta. Checking AM states. 2024-11-20T08:48:46,236 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T08:48:46,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732092525591 assigned. 2024-11-20T08:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732092525591', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960., hostname=be1eb2620e0e,45375,1732092524555, seqNum=2] 2024-11-20T08:48:46,245 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7798a5357e73d077d13b95bd29adb960 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T08:48:46,256 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4d330b5e784e44999c6e7bdfdbff4fef 2024-11-20T08:48:46,272 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/.tmp/family/4d330b5e784e44999c6e7bdfdbff4fef is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T08:48:46,273 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/.tmp/family/4d330b5e784e44999c6e7bdfdbff4fef 2024-11-20T08:48:46,276 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/.tmp/family/4d330b5e784e44999c6e7bdfdbff4fef as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef 2024-11-20T08:48:46,277 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef, delay=921 ms 2024-11-20T08:48:46,278 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T08:48:46,279 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 7798a5357e73d077d13b95bd29adb960 in 34ms, sequenceid=6, compaction requested=false 2024-11-20T08:48:46,279 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732092525591' 2024-11-20T08:48:46,279 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7798a5357e73d077d13b95bd29adb960: 2024-11-20T08:48:46,635 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T08:48:47,199 DEBUG [hfile-prefetch-1732092523867 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4d330b5e784e44999c6e7bdfdbff4fef_0. 2024-11-20T08:48:47,200 DEBUG [hfile-prefetch-1732092523867 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4d330b5e784e44999c6e7bdfdbff4fef_117. 2024-11-20T08:48:47,200 DEBUG [hfile-prefetch-1732092523867 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef, with total blocks 2, and data blocks 1 2024-11-20T08:48:47,200 DEBUG [hfile-prefetch-1732092523867 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T08:48:47,200 DEBUG [hfile-prefetch-1732092523867 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T08:48:47,200 DEBUG [hfile-prefetch-1732092523867 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef. Will sleep 100ms and try the verification again. 2024-11-20T08:48:47,232 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T08:48:47,232 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T08:48:47,232 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:47,232 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:47,236 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T08:48:47,236 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732092527232 2024-11-20T08:48:47,236 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T08:48:47,280 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;be1eb2620e0e:45375,5,FailOnTimeoutGroup] 2024-11-20T08:48:47,280 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'be1eb2620e0e,45375,1732092524555' ***** 2024-11-20T08:48:47,280 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-20T08:48:47,280 INFO [RS:1;be1eb2620e0e:45375 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T08:48:47,281 INFO [RS:1;be1eb2620e0e:45375 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.persistence 2024-11-20T08:48:47,281 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T08:48:47,281 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T08:48:47,281 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T08:48:47,281 INFO [RS:1;be1eb2620e0e:45375 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T08:48:47,281 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T08:48:47,291 DEBUG [RS:1;be1eb2620e0e:45375 {}] bucket.BucketCache(1487): Thread RS:1;be1eb2620e0e:45375 started persisting bucket cache to file 2024-11-20T08:48:47,291 DEBUG [RS:1;be1eb2620e0e:45375 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:47,291 DEBUG [RS:1;be1eb2620e0e:45375 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:47,294 DEBUG [RS:1;be1eb2620e0e:45375 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T08:48:47,295 DEBUG [RS:1;be1eb2620e0e:45375 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732092527291 2024-11-20T08:48:47,295 DEBUG [RS:1;be1eb2620e0e:45375 {}] bucket.BucketCache(1509): Thread RS:1;be1eb2620e0e:45375 finished persisting bucket cache to file, renaming 2024-11-20T08:48:47,295 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T08:48:47,295 INFO [RS:1;be1eb2620e0e:45375 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T08:48:47,295 INFO [RS:1;be1eb2620e0e:45375 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T08:48:47,295 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T08:48:47,295 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(3091): Received CLOSE for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:47,295 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(959): stopping server be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:47,295 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T08:48:47,296 INFO [RS:1;be1eb2620e0e:45375 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;be1eb2620e0e:45375. 2024-11-20T08:48:47,296 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T08:48:47,296 DEBUG [RS:1;be1eb2620e0e:45375 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:48:47,296 DEBUG [RS:1;be1eb2620e0e:45375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:47,296 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7798a5357e73d077d13b95bd29adb960, disabling compactions & flushes 2024-11-20T08:48:47,296 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:47,296 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T08:48:47,296 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T08:48:47,296 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:47,296 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T08:48:47,296 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T08:48:47,296 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. after waiting 0 ms 2024-11-20T08:48:47,296 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:47,296 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T08:48:47,296 DEBUG [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(1325): Online Regions={7798a5357e73d077d13b95bd29adb960=testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T08:48:47,296 DEBUG [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:47,296 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T08:48:47,297 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T08:48:47,297 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T08:48:47,297 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T08:48:47,297 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T08:48:47,297 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T08:48:47,297 DEBUG [StoreFileCloser-7798a5357e73d077d13b95bd29adb960-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef 2024-11-20T08:48:47,297 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-20T08:48:47,299 WARN [hfile-prefetch-1732092523867 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-20T08:48:47,299 DEBUG [hfile-prefetch-1732092523867 {}] hfile.PrefetchExecutor(121): Prefetch completed for 4d330b5e784e44999c6e7bdfdbff4fef 2024-11-20T08:48:47,302 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T08:48:47,302 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:47,302 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7798a5357e73d077d13b95bd29adb960: Waiting for close lock at 1732092527296Running coprocessor pre-close hooks at 1732092527296Disabling compacts and flushes for region at 1732092527296Disabling writes for close at 1732092527296Writing region close event to WAL at 1732092527297 (+1 ms)Running coprocessor post-close hooks at 1732092527302 (+5 ms)Closed at 1732092527302 2024-11-20T08:48:47,303 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:47,310 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f011cb5448064f58951c569dda28fa36 2024-11-20T08:48:47,324 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/info/f011cb5448064f58951c569dda28fa36 is 209, key is testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960./info:regioninfo/1732092525958/Put/seqid=0 2024-11-20T08:48:47,325 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/info/f011cb5448064f58951c569dda28fa36 2024-11-20T08:48:47,334 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d32c90264aa34167bdb824aba4c70d5a 2024-11-20T08:48:47,343 INFO [regionserver/be1eb2620e0e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:47,346 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/ns/d32c90264aa34167bdb824aba4c70d5a is 43, key is default/ns:d/1732092525510/Put/seqid=0 2024-11-20T08:48:47,347 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/ns/d32c90264aa34167bdb824aba4c70d5a 2024-11-20T08:48:47,356 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 557fff19ee8d4ce0b70bb2bc4e981a4d 2024-11-20T08:48:47,368 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/table/557fff19ee8d4ce0b70bb2bc4e981a4d is 80, key is testBlockEvictionOnGracefulStop1732092525591/table:state/1732092525969/Put/seqid=0 2024-11-20T08:48:47,369 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/table/557fff19ee8d4ce0b70bb2bc4e981a4d 2024-11-20T08:48:47,372 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/info/f011cb5448064f58951c569dda28fa36 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/f011cb5448064f58951c569dda28fa36 2024-11-20T08:48:47,373 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/f011cb5448064f58951c569dda28fa36, delay=928 ms 2024-11-20T08:48:47,374 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/f011cb5448064f58951c569dda28fa36, entries=10, sequenceid=11, filesize=7.0 K 2024-11-20T08:48:47,374 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/ns/d32c90264aa34167bdb824aba4c70d5a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/d32c90264aa34167bdb824aba4c70d5a 2024-11-20T08:48:47,375 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/d32c90264aa34167bdb824aba4c70d5a, delay=976 ms 2024-11-20T08:48:47,376 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/d32c90264aa34167bdb824aba4c70d5a, entries=2, sequenceid=11, filesize=5.0 K 2024-11-20T08:48:47,377 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/table/557fff19ee8d4ce0b70bb2bc4e981a4d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/557fff19ee8d4ce0b70bb2bc4e981a4d 2024-11-20T08:48:47,378 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/557fff19ee8d4ce0b70bb2bc4e981a4d, delay=968 ms 2024-11-20T08:48:47,379 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/557fff19ee8d4ce0b70bb2bc4e981a4d, entries=2, sequenceid=11, filesize=5.3 K 2024-11-20T08:48:47,379 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=11, compaction requested=false 2024-11-20T08:48:47,380 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/f011cb5448064f58951c569dda28fa36 2024-11-20T08:48:47,381 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/d32c90264aa34167bdb824aba4c70d5a 2024-11-20T08:48:47,381 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/557fff19ee8d4ce0b70bb2bc4e981a4d 2024-11-20T08:48:47,386 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-20T08:48:47,386 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T08:48:47,386 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T08:48:47,386 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732092527296Running coprocessor pre-close hooks at 1732092527296Disabling compacts and flushes for region at 1732092527297 (+1 ms)Disabling writes for close at 1732092527297Obtaining lock to block concurrent updates at 1732092527297Preparing flush snapshotting stores in 1588230740 at 1732092527297Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732092527297Flushing stores of hbase:meta,,1.1588230740 at 1732092527298 (+1 ms)Flushing 1588230740/info: creating writer at 1732092527298Flushing 1588230740/info: appending metadata at 1732092527323 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732092527323Flushing 1588230740/ns: creating writer at 1732092527327 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732092527346 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732092527346Flushing 1588230740/table: creating writer at 1732092527349 (+3 ms)Flushing 1588230740/table: appending metadata at 1732092527368 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732092527368Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29054a3: reopening flushed file at 1732092527371 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39325b7b: reopening flushed file at 1732092527374 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd5057b: reopening flushed file at 1732092527376 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=11, compaction requested=false at 1732092527379 (+3 ms)Writing region close event to WAL at 1732092527381 (+2 ms)Running coprocessor post-close hooks at 1732092527386 (+5 ms)Closed at 1732092527386 2024-11-20T08:48:47,386 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T08:48:47,497 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(976): stopping server be1eb2620e0e,45375,1732092524555; all regions closed. 2024-11-20T08:48:47,499 DEBUG [RS:1;be1eb2620e0e:45375 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs 2024-11-20T08:48:47,499 INFO [RS:1;be1eb2620e0e:45375 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C45375%2C1732092524555.meta:.meta(num 1732092525462) 2024-11-20T08:48:47,500 DEBUG [RS:1;be1eb2620e0e:45375 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs 2024-11-20T08:48:47,500 INFO [RS:1;be1eb2620e0e:45375 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C45375%2C1732092524555:(num 1732092525079) 2024-11-20T08:48:47,500 DEBUG [RS:1;be1eb2620e0e:45375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:47,500 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:47,501 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T08:48:47,501 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.ChoreService(370): Chore service for: regionserver/be1eb2620e0e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T08:48:47,501 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T08:48:47,501 INFO [regionserver/be1eb2620e0e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T08:48:47,501 INFO [RS:1;be1eb2620e0e:45375 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45375 2024-11-20T08:48:47,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T08:48:47,506 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T08:48:47,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/be1eb2620e0e,45375,1732092524555 2024-11-20T08:48:47,508 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [be1eb2620e0e,45375,1732092524555] 2024-11-20T08:48:47,510 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/be1eb2620e0e,45375,1732092524555 already deleted, retry=false 2024-11-20T08:48:47,510 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of be1eb2620e0e,45375,1732092524555 on be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:47,514 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure be1eb2620e0e,45375,1732092524555, splitWal=true, meta=true 2024-11-20T08:48:47,516 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for be1eb2620e0e,45375,1732092524555 (carryingMeta=true) be1eb2620e0e,45375,1732092524555/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@14fe2f60[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T08:48:47,516 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure be1eb2620e0e,45375,1732092524555, splitWal=true, meta=true 2024-11-20T08:48:47,517 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure be1eb2620e0e,45375,1732092524555, splitWal=true, meta=true, isMeta: true 2024-11-20T08:48:47,518 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555-splitting 2024-11-20T08:48:47,518 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555-splitting dir is empty, no logs to split. 2024-11-20T08:48:47,518 INFO [PEWorker-2 {}] master.SplitWALManager(105): be1eb2620e0e,45375,1732092524555 WAL count=0, meta=true 2024-11-20T08:48:47,520 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555-splitting dir is empty, no logs to split. 2024-11-20T08:48:47,520 INFO [PEWorker-2 {}] master.SplitWALManager(105): be1eb2620e0e,45375,1732092524555 WAL count=0, meta=true 2024-11-20T08:48:47,520 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if be1eb2620e0e,45375,1732092524555 WAL splitting is done? wals=0, meta=true 2024-11-20T08:48:47,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T08:48:47,523 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T08:48:47,523 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T08:48:47,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:47,608 INFO [RS:1;be1eb2620e0e:45375 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T08:48:47,609 INFO [RS:1;be1eb2620e0e:45375 {}] regionserver.HRegionServer(1031): Exiting; stopping=be1eb2620e0e,45375,1732092524555; zookeeper connection closed. 2024-11-20T08:48:47,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45375-0x100141e30850002, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:47,609 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b96ee90 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b96ee90 2024-11-20T08:48:47,674 DEBUG [be1eb2620e0e:40345 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T08:48:47,674 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:47,675 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as be1eb2620e0e,43559,1732092524519, state=OPENING 2024-11-20T08:48:47,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:47,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:47,677 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T08:48:47,677 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:47,677 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:47,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=be1eb2620e0e,43559,1732092524519}] 2024-11-20T08:48:47,798 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/be1eb2620e0e:0 server-side Connection retries=45 2024-11-20T08:48:47,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:47,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:47,798 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T08:48:47,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:48:47,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T08:48:47,798 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T08:48:47,798 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T08:48:47,799 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35761 2024-11-20T08:48:47,801 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35761 connecting to ZooKeeper ensemble=127.0.0.1:62105 2024-11-20T08:48:47,801 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:47,801 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:47,805 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-20T08:48:47,805 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:357610x0, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-20T08:48:47,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:357610x0, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T08:48:47,805 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35761-0x100141e30850003 connected 2024-11-20T08:48:47,806 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T08:48:47,807 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.cache 2024-11-20T08:48:47,807 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T08:48:47,807 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T08:48:47,807 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T08:48:47,808 INFO [Thread-208 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T08:48:47,809 INFO [Thread-208 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-20T08:48:47,810 INFO [Thread-208 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-20T08:48:47,815 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T08:48:47,815 DEBUG [Thread-208 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-20T08:48:47,815 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T08:48:47,816 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T08:48:47,816 INFO [Thread-208 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-20T08:48:47,816 INFO [Thread-208 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.persistence is complete. 2024-11-20T08:48:47,816 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T08:48:47,818 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T08:48:47,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35761 2024-11-20T08:48:47,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35761 2024-11-20T08:48:47,824 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35761 2024-11-20T08:48:47,825 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35761 2024-11-20T08:48:47,825 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35761 2024-11-20T08:48:47,829 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(746): ClusterId : 2798e9c4-7db4-421b-ae34-4225e7304b5d 2024-11-20T08:48:47,829 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T08:48:47,830 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T08:48:47,831 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T08:48:47,831 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T08:48:47,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37353, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T08:48:47,834 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T08:48:47,834 DEBUG [RS:2;be1eb2620e0e:35761 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24875574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=be1eb2620e0e/172.17.0.2:0 2024-11-20T08:48:47,835 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T08:48:47,835 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:47,835 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T08:48:47,836 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=be1eb2620e0e%2C43559%2C1732092524519.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,43559,1732092524519, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs, maxLogs=55 2024-11-20T08:48:47,848 DEBUG [RS:2;be1eb2620e0e:35761 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;be1eb2620e0e:35761 2024-11-20T08:48:47,848 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T08:48:47,848 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T08:48:47,848 DEBUG [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T08:48:47,849 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(2659): reportForDuty to master=be1eb2620e0e,40345,1732092524465 with port=35761, startcode=1732092527797 2024-11-20T08:48:47,849 DEBUG [RS:2;be1eb2620e0e:35761 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T08:48:47,851 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52843, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T08:48:47,852 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40345 {}] master.ServerManager(363): Checking decommissioned status of RegionServer be1eb2620e0e,35761,1732092527797 2024-11-20T08:48:47,852 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40345 {}] master.ServerManager(517): Registering regionserver=be1eb2620e0e,35761,1732092527797 2024-11-20T08:48:47,854 DEBUG [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239 2024-11-20T08:48:47,854 DEBUG [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T08:48:47,854 DEBUG [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T08:48:47,854 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,43559,1732092524519/be1eb2620e0e%2C43559%2C1732092524519.meta.1732092527837.meta 2024-11-20T08:48:47,854 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:47,855 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:47,855 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T08:48:47,855 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T08:48:47,855 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T08:48:47,855 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T08:48:47,855 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:47,856 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T08:48:47,856 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T08:48:47,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T08:48:47,857 DEBUG [RS:2;be1eb2620e0e:35761 {}] zookeeper.ZKUtil(111): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/be1eb2620e0e,35761,1732092527797 2024-11-20T08:48:47,857 WARN [RS:2;be1eb2620e0e:35761 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T08:48:47,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T08:48:47,857 INFO [RS:2;be1eb2620e0e:35761 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:48:47,857 DEBUG [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,35761,1732092527797 2024-11-20T08:48:47,857 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [be1eb2620e0e,35761,1732092527797] 2024-11-20T08:48:47,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T08:48:47,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:47,861 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/f011cb5448064f58951c569dda28fa36, delay=938 ms 2024-11-20T08:48:47,863 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/f011cb5448064f58951c569dda28fa36 2024-11-20T08:48:47,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:47,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T08:48:47,864 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T08:48:47,864 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:47,865 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T08:48:47,865 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/d32c90264aa34167bdb824aba4c70d5a, delay=987 ms 2024-11-20T08:48:47,867 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/d32c90264aa34167bdb824aba4c70d5a 2024-11-20T08:48:47,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:47,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T08:48:47,867 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T08:48:47,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:47,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:47,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T08:48:47,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T08:48:47,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:47,870 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T08:48:47,870 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/557fff19ee8d4ce0b70bb2bc4e981a4d, delay=967 ms 2024-11-20T08:48:47,871 INFO [RS:2;be1eb2620e0e:35761 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T08:48:47,871 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,871 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/557fff19ee8d4ce0b70bb2bc4e981a4d 2024-11-20T08:48:47,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:48:47,871 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T08:48:47,872 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T08:48:47,872 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:47,872 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T08:48:47,872 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,872 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/be1eb2620e0e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:48:47,873 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:47,874 DEBUG [RS:2;be1eb2620e0e:35761 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:48:47,874 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,874 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,874 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T08:48:47,874 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,874 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,874 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,874 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,35761,1732092527797-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T08:48:47,875 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66232303, jitterRate=-0.013061776757240295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T08:48:47,875 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T08:48:47,876 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732092527856Writing region info on filesystem at 1732092527856Initializing all the Stores at 1732092527856Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092527856Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092527856Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092527856Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732092527857 (+1 ms)Cleaning up temporary data from old regions at 1732092527873 (+16 ms)Running coprocessor post-open hooks at 1732092527875 (+2 ms)Region opened successfully at 1732092527876 (+1 ms) 2024-11-20T08:48:47,876 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732092527829 2024-11-20T08:48:47,879 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T08:48:47,879 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T08:48:47,879 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:47,880 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as be1eb2620e0e,43559,1732092524519, state=OPEN 2024-11-20T08:48:47,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:47,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:47,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:48:47,882 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:47,882 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:47,882 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:47,882 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:48:47,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T08:48:47,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=be1eb2620e0e,43559,1732092524519 in 205 msec 2024-11-20T08:48:47,886 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T08:48:47,886 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 363 msec 2024-11-20T08:48:47,887 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): be1eb2620e0e,45375,1732092524555 had 2 regions 2024-11-20T08:48:47,888 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure be1eb2620e0e,45375,1732092524555, splitWal=true, meta=true, isMeta: false 2024-11-20T08:48:47,888 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555-splitting dir is empty, no logs to split. 2024-11-20T08:48:47,888 INFO [PEWorker-3 {}] master.SplitWALManager(105): be1eb2620e0e,45375,1732092524555 WAL count=0, meta=false 2024-11-20T08:48:47,889 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,45375,1732092524555-splitting dir is empty, no logs to split. 2024-11-20T08:48:47,889 INFO [PEWorker-3 {}] master.SplitWALManager(105): be1eb2620e0e,45375,1732092524555 WAL count=0, meta=false 2024-11-20T08:48:47,889 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if be1eb2620e0e,45375,1732092524555 WAL splitting is done? wals=0, meta=false 2024-11-20T08:48:47,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN}] 2024-11-20T08:48:47,891 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN 2024-11-20T08:48:47,892 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T08:48:47,893 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T08:48:47,893 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,35761,1732092527797-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,893 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,893 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.Replication(171): be1eb2620e0e,35761,1732092527797 started 2024-11-20T08:48:47,909 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:48:47,909 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(1482): Serving as be1eb2620e0e,35761,1732092527797, RpcServer on be1eb2620e0e/172.17.0.2:35761, sessionid=0x100141e30850003 2024-11-20T08:48:47,909 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T08:48:47,909 DEBUG [RS:2;be1eb2620e0e:35761 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager be1eb2620e0e,35761,1732092527797 2024-11-20T08:48:47,909 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,35761,1732092527797' 2024-11-20T08:48:47,909 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T08:48:47,910 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T08:48:47,911 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T08:48:47,911 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T08:48:47,911 DEBUG [RS:2;be1eb2620e0e:35761 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager be1eb2620e0e,35761,1732092527797 2024-11-20T08:48:47,911 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,35761,1732092527797' 2024-11-20T08:48:47,911 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T08:48:47,911 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T08:48:47,912 DEBUG [RS:2;be1eb2620e0e:35761 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T08:48:47,912 INFO [RS:2;be1eb2620e0e:35761 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T08:48:47,912 INFO [RS:2;be1eb2620e0e:35761 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T08:48:48,013 INFO [RS:2;be1eb2620e0e:35761 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T08:48:48,013 INFO [RS:2;be1eb2620e0e:35761 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=be1eb2620e0e%2C35761%2C1732092527797, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,35761,1732092527797, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs, maxLogs=55 2024-11-20T08:48:48,033 INFO [RS:2;be1eb2620e0e:35761 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/WALs/be1eb2620e0e,35761,1732092527797/be1eb2620e0e%2C35761%2C1732092527797.1732092528014 2024-11-20T08:48:48,034 DEBUG [RS:2;be1eb2620e0e:35761 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T08:48:48,043 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(204): Hosts are {be1eb2620e0e=0} racks are {/default-rack=0} 2024-11-20T08:48:48,043 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T08:48:48,043 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T08:48:48,043 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T08:48:48,043 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T08:48:48,043 INFO [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T08:48:48,043 INFO [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T08:48:48,043 DEBUG [be1eb2620e0e:40345 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T08:48:48,043 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7798a5357e73d077d13b95bd29adb960, regionState=OPENING, regionLocation=be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:48,045 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server be1eb2620e0e:45375 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: be1eb2620e0e/172.17.0.2:45375 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:48:48,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,45375,1732092524555, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,45375,1732092524555, seqNum=-1, error=java.net.ConnectException: Call to address=be1eb2620e0e:45375 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: be1eb2620e0e/172.17.0.2:45375 2024-11-20T08:48:48,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,45375,1732092524555, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T08:48:48,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,45375,1732092524555, seqNum=-1 from cache 2024-11-20T08:48:48,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address be1eb2620e0e:45375 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: be1eb2620e0e/172.17.0.2:45375 2024-11-20T08:48:48,151 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:48:48,151 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,43559,1732092524519, seqNum=-1] 2024-11-20T08:48:48,152 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:48:48,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40375, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:48:48,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN because future has completed 2024-11-20T08:48:48,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7798a5357e73d077d13b95bd29adb960, server=be1eb2620e0e,43559,1732092524519}] 2024-11-20T08:48:48,311 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:48,311 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 7798a5357e73d077d13b95bd29adb960, NAME => 'testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:48:48,312 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732092525591 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,312 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:48:48,312 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,313 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,313 INFO [StoreOpener-7798a5357e73d077d13b95bd29adb960-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,314 INFO [StoreOpener-7798a5357e73d077d13b95bd29adb960-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7798a5357e73d077d13b95bd29adb960 columnFamilyName family 2024-11-20T08:48:48,314 DEBUG [StoreOpener-7798a5357e73d077d13b95bd29adb960-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:48:48,316 DEBUG [StoreFileOpener-7798a5357e73d077d13b95bd29adb960-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef, delay=980 ms 2024-11-20T08:48:48,317 DEBUG [StoreOpener-7798a5357e73d077d13b95bd29adb960-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef 2024-11-20T08:48:48,317 INFO [StoreOpener-7798a5357e73d077d13b95bd29adb960-1 {}] regionserver.HStore(327): Store=7798a5357e73d077d13b95bd29adb960/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:48:48,317 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,318 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,318 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,318 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,318 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,320 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,320 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 7798a5357e73d077d13b95bd29adb960; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68015011, jitterRate=0.013502642512321472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T08:48:48,320 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,321 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 7798a5357e73d077d13b95bd29adb960: Running coprocessor pre-open hook at 1732092528313Writing region info on filesystem at 1732092528313Initializing all the Stores at 1732092528313Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732092528313Cleaning up temporary data from old regions at 1732092528319 (+6 ms)Running coprocessor post-open hooks at 1732092528320 (+1 ms)Region opened successfully at 1732092528321 (+1 ms) 2024-11-20T08:48:48,322 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960., pid=11, masterSystemTime=1732092528309 2024-11-20T08:48:48,325 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:48,325 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:48,326 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7798a5357e73d077d13b95bd29adb960, regionState=OPEN, openSeqNum=10, regionLocation=be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:48,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7798a5357e73d077d13b95bd29adb960, server=be1eb2620e0e,43559,1732092524519 because future has completed 2024-11-20T08:48:48,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T08:48:48,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 7798a5357e73d077d13b95bd29adb960, server=be1eb2620e0e,43559,1732092524519 in 173 msec 2024-11-20T08:48:48,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-20T08:48:48,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732092525591, region=7798a5357e73d077d13b95bd29adb960, ASSIGN in 440 msec 2024-11-20T08:48:48,332 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server be1eb2620e0e,45375,1732092524555 after splitting done 2024-11-20T08:48:48,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure be1eb2620e0e,45375,1732092524555, splitWal=true, meta=true in 821 msec 2024-11-20T08:48:48,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T08:48:48,409 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T08:48:48,409 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:48:48,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:48,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:48,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:48:48,410 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T08:48:48,410 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=267234785, stopped=false 2024-11-20T08:48:48,410 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=be1eb2620e0e,40345,1732092524465 2024-11-20T08:48:48,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:48,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:48,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T08:48:48,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:48,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:48,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:48,413 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T08:48:48,413 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T08:48:48,413 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:48,413 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:48,413 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:48:48,413 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:48:48,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:48,414 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'be1eb2620e0e,43559,1732092524519' ***** 2024-11-20T08:48:48,414 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T08:48:48,414 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'be1eb2620e0e,35761,1732092527797' ***** 2024-11-20T08:48:48,414 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T08:48:48,414 INFO [RS:0;be1eb2620e0e:43559 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T08:48:48,414 INFO [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T08:48:48,414 INFO [RS:0;be1eb2620e0e:43559 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.persistence 2024-11-20T08:48:48,414 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T08:48:48,414 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T08:48:48,414 INFO [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83af9ebb-ad3e-cb7f-c29b-6ad6f94258e5/bucket.persistence 2024-11-20T08:48:48,414 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T08:48:48,414 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T08:48:48,414 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T08:48:48,414 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T08:48:48,414 INFO [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T08:48:48,414 INFO [RS:0;be1eb2620e0e:43559 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T08:48:48,415 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T08:48:48,415 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T08:48:48,425 DEBUG [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1487): Thread RS:2;be1eb2620e0e:35761 started persisting bucket cache to file 2024-11-20T08:48:48,425 DEBUG [RS:0;be1eb2620e0e:43559 {}] bucket.BucketCache(1487): Thread RS:0;be1eb2620e0e:43559 started persisting bucket cache to file 2024-11-20T08:48:48,425 DEBUG [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:48,425 DEBUG [RS:0;be1eb2620e0e:43559 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T08:48:48,425 DEBUG [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:48,425 DEBUG [RS:0;be1eb2620e0e:43559 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T08:48:48,428 DEBUG [RS:0;be1eb2620e0e:43559 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T08:48:48,428 DEBUG [RS:0;be1eb2620e0e:43559 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732092528425 2024-11-20T08:48:48,428 DEBUG [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T08:48:48,428 DEBUG [RS:0;be1eb2620e0e:43559 {}] bucket.BucketCache(1509): Thread RS:0;be1eb2620e0e:43559 finished persisting bucket cache to file, renaming 2024-11-20T08:48:48,428 DEBUG [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732092528425 2024-11-20T08:48:48,428 DEBUG [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1509): Thread RS:2;be1eb2620e0e:35761 finished persisting bucket cache to file, renaming 2024-11-20T08:48:48,429 WARN [RS:2;be1eb2620e0e:35761 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-20T08:48:48,429 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T08:48:48,429 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T08:48:48,429 INFO [RS:0;be1eb2620e0e:43559 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T08:48:48,429 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T08:48:48,429 INFO [RS:0;be1eb2620e0e:43559 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T08:48:48,429 INFO [RS:2;be1eb2620e0e:35761 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T08:48:48,429 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T08:48:48,429 INFO [RS:2;be1eb2620e0e:35761 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T08:48:48,429 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(959): stopping server be1eb2620e0e,35761,1732092527797 2024-11-20T08:48:48,429 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(3091): Received CLOSE for 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,429 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T08:48:48,429 INFO [RS:2;be1eb2620e0e:35761 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;be1eb2620e0e:35761. 2024-11-20T08:48:48,429 DEBUG [RS:2;be1eb2620e0e:35761 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:48:48,429 DEBUG [RS:2;be1eb2620e0e:35761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:48,429 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(959): stopping server be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:48,429 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T08:48:48,429 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(976): stopping server be1eb2620e0e,35761,1732092527797; all regions closed. 2024-11-20T08:48:48,429 INFO [RS:0;be1eb2620e0e:43559 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;be1eb2620e0e:43559. 2024-11-20T08:48:48,429 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T08:48:48,429 DEBUG [RS:0;be1eb2620e0e:43559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:48:48,430 DEBUG [RS:0;be1eb2620e0e:43559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:48,430 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7798a5357e73d077d13b95bd29adb960, disabling compactions & flushes 2024-11-20T08:48:48,430 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:48,430 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T08:48:48,430 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T08:48:48,430 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T08:48:48,430 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:48,430 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T08:48:48,430 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. after waiting 0 ms 2024-11-20T08:48:48,430 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:48,430 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T08:48:48,430 DEBUG [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(1325): Online Regions={7798a5357e73d077d13b95bd29adb960=testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T08:48:48,430 DEBUG [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7798a5357e73d077d13b95bd29adb960 2024-11-20T08:48:48,430 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T08:48:48,430 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T08:48:48,431 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T08:48:48,431 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T08:48:48,431 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T08:48:48,431 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T08:48:48,431 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-20T08:48:48,431 DEBUG [StoreFileCloser-7798a5357e73d077d13b95bd29adb960-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/family/4d330b5e784e44999c6e7bdfdbff4fef 2024-11-20T08:48:48,431 DEBUG [RS:2;be1eb2620e0e:35761 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs 2024-11-20T08:48:48,431 INFO [RS:2;be1eb2620e0e:35761 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C35761%2C1732092527797:(num 1732092528014) 2024-11-20T08:48:48,431 DEBUG [RS:2;be1eb2620e0e:35761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:48,431 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:48,431 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T08:48:48,432 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.ChoreService(370): Chore service for: regionserver/be1eb2620e0e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T08:48:48,432 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T08:48:48,432 INFO [regionserver/be1eb2620e0e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T08:48:48,432 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T08:48:48,432 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T08:48:48,432 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T08:48:48,432 INFO [RS:2;be1eb2620e0e:35761 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35761 2024-11-20T08:48:48,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T08:48:48,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/be1eb2620e0e,35761,1732092527797 2024-11-20T08:48:48,437 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T08:48:48,437 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/default/testBlockEvictionOnGracefulStop1732092525591/7798a5357e73d077d13b95bd29adb960/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T08:48:48,438 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [be1eb2620e0e,35761,1732092527797] 2024-11-20T08:48:48,438 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:48,438 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7798a5357e73d077d13b95bd29adb960: Waiting for close lock at 1732092528429Running coprocessor pre-close hooks at 1732092528429Disabling compacts and flushes for region at 1732092528429Disabling writes for close at 1732092528430 (+1 ms)Writing region close event to WAL at 1732092528431 (+1 ms)Running coprocessor post-close hooks at 1732092528438 (+7 ms)Closed at 1732092528438 2024-11-20T08:48:48,438 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960. 2024-11-20T08:48:48,440 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/be1eb2620e0e,35761,1732092527797 already deleted, retry=false 2024-11-20T08:48:48,440 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; be1eb2620e0e,35761,1732092527797 expired; onlineServers=1 2024-11-20T08:48:48,443 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 183af5b407a9472598d013218f9d806c 2024-11-20T08:48:48,444 INFO [regionserver/be1eb2620e0e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:48,457 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/info/183af5b407a9472598d013218f9d806c is 209, key is testBlockEvictionOnGracefulStop1732092525591,,1732092525595.7798a5357e73d077d13b95bd29adb960./info:regioninfo/1732092528325/Put/seqid=0 2024-11-20T08:48:48,458 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/info/183af5b407a9472598d013218f9d806c 2024-11-20T08:48:48,460 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/.tmp/info/183af5b407a9472598d013218f9d806c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/183af5b407a9472598d013218f9d806c 2024-11-20T08:48:48,461 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/183af5b407a9472598d013218f9d806c, delay=915 ms 2024-11-20T08:48:48,462 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/183af5b407a9472598d013218f9d806c, entries=8, sequenceid=19, filesize=6.6 K 2024-11-20T08:48:48,463 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false 2024-11-20T08:48:48,464 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/f011cb5448064f58951c569dda28fa36 2024-11-20T08:48:48,464 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/info/183af5b407a9472598d013218f9d806c 2024-11-20T08:48:48,465 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/ns/d32c90264aa34167bdb824aba4c70d5a 2024-11-20T08:48:48,465 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/table/557fff19ee8d4ce0b70bb2bc4e981a4d 2024-11-20T08:48:48,470 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-20T08:48:48,470 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T08:48:48,470 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T08:48:48,470 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732092528430Running coprocessor pre-close hooks at 1732092528430Disabling compacts and flushes for region at 1732092528430Disabling writes for close at 1732092528431 (+1 ms)Obtaining lock to block concurrent updates at 1732092528431Preparing flush snapshotting stores in 1588230740 at 1732092528431Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732092528431Flushing stores of hbase:meta,,1.1588230740 at 1732092528432 (+1 ms)Flushing 1588230740/info: creating writer at 1732092528432Flushing 1588230740/info: appending metadata at 1732092528457 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732092528457Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18f7f222: reopening flushed file at 1732092528460 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false at 1732092528463 (+3 ms)Writing region close event to WAL at 1732092528466 (+3 ms)Running coprocessor post-close hooks at 1732092528470 (+4 ms)Closed at 1732092528470 2024-11-20T08:48:48,470 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T08:48:48,477 INFO [regionserver/be1eb2620e0e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:48,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:48,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35761-0x100141e30850003, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:48,540 INFO [RS:2;be1eb2620e0e:35761 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T08:48:48,540 INFO [RS:2;be1eb2620e0e:35761 {}] regionserver.HRegionServer(1031): Exiting; stopping=be1eb2620e0e,35761,1732092527797; zookeeper connection closed. 2024-11-20T08:48:48,541 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@24a998c0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@24a998c0 2024-11-20T08:48:48,631 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(976): stopping server be1eb2620e0e,43559,1732092524519; all regions closed. 2024-11-20T08:48:48,633 DEBUG [RS:0;be1eb2620e0e:43559 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs 2024-11-20T08:48:48,634 INFO [RS:0;be1eb2620e0e:43559 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C43559%2C1732092524519.meta:.meta(num 1732092527837) 2024-11-20T08:48:48,635 DEBUG [RS:0;be1eb2620e0e:43559 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/oldWALs 2024-11-20T08:48:48,635 INFO [RS:0;be1eb2620e0e:43559 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C43559%2C1732092524519:(num 1732092525085) 2024-11-20T08:48:48,635 DEBUG [RS:0;be1eb2620e0e:43559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:48:48,635 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T08:48:48,640 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T08:48:48,640 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.ChoreService(370): Chore service for: regionserver/be1eb2620e0e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T08:48:48,640 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T08:48:48,640 INFO [regionserver/be1eb2620e0e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T08:48:48,641 INFO [RS:0;be1eb2620e0e:43559 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43559 2024-11-20T08:48:48,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/be1eb2620e0e,43559,1732092524519 2024-11-20T08:48:48,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T08:48:48,643 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T08:48:48,645 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [be1eb2620e0e,43559,1732092524519] 2024-11-20T08:48:48,646 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/be1eb2620e0e,43559,1732092524519 already deleted, retry=false 2024-11-20T08:48:48,647 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; be1eb2620e0e,43559,1732092524519 expired; onlineServers=0 2024-11-20T08:48:48,647 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'be1eb2620e0e,40345,1732092524465' ***** 2024-11-20T08:48:48,647 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T08:48:48,647 INFO [M:0;be1eb2620e0e:40345 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T08:48:48,647 INFO [M:0;be1eb2620e0e:40345 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T08:48:48,647 DEBUG [M:0;be1eb2620e0e:40345 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T08:48:48,647 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T08:48:48,647 DEBUG [M:0;be1eb2620e0e:40345 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T08:48:48,647 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.large.0-1732092524834 {}] cleaner.HFileCleaner(306): Exit Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.large.0-1732092524834,5,FailOnTimeoutGroup] 2024-11-20T08:48:48,647 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.small.0-1732092524836 {}] cleaner.HFileCleaner(306): Exit Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.small.0-1732092524836,5,FailOnTimeoutGroup] 2024-11-20T08:48:48,648 INFO [M:0;be1eb2620e0e:40345 {}] hbase.ChoreService(370): Chore service for: master/be1eb2620e0e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T08:48:48,648 INFO [M:0;be1eb2620e0e:40345 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T08:48:48,648 DEBUG [M:0;be1eb2620e0e:40345 {}] master.HMaster(1795): Stopping service threads 2024-11-20T08:48:48,648 INFO [M:0;be1eb2620e0e:40345 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T08:48:48,648 INFO [M:0;be1eb2620e0e:40345 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T08:48:48,648 INFO [M:0;be1eb2620e0e:40345 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T08:48:48,648 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T08:48:48,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T08:48:48,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:48:48,649 DEBUG [M:0;be1eb2620e0e:40345 {}] zookeeper.ZKUtil(347): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T08:48:48,649 WARN [M:0;be1eb2620e0e:40345 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T08:48:48,649 INFO [M:0;be1eb2620e0e:40345 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/.lastflushedseqids 2024-11-20T08:48:48,658 INFO [M:0;be1eb2620e0e:40345 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T08:48:48,658 INFO [M:0;be1eb2620e0e:40345 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T08:48:48,658 DEBUG [M:0;be1eb2620e0e:40345 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T08:48:48,658 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:48,658 DEBUG [M:0;be1eb2620e0e:40345 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:48,659 DEBUG [M:0;be1eb2620e0e:40345 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T08:48:48,659 DEBUG [M:0;be1eb2620e0e:40345 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:48,659 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-20T08:48:48,669 DEBUG [M:0;be1eb2620e0e:40345 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5d81ad22834844e1ba3c34e2f22ca752 2024-11-20T08:48:48,683 DEBUG [M:0;be1eb2620e0e:40345 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d81ad22834844e1ba3c34e2f22ca752 is 82, key is hbase:meta,,1/info:regioninfo/1732092527879/Put/seqid=0 2024-11-20T08:48:48,684 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d81ad22834844e1ba3c34e2f22ca752 2024-11-20T08:48:48,692 DEBUG [M:0;be1eb2620e0e:40345 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2f549348be7b4dac9a6d0cb4a94290e4 2024-11-20T08:48:48,706 DEBUG [M:0;be1eb2620e0e:40345 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2f549348be7b4dac9a6d0cb4a94290e4 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732092525974/Put/seqid=0 2024-11-20T08:48:48,707 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2f549348be7b4dac9a6d0cb4a94290e4 2024-11-20T08:48:48,715 DEBUG [M:0;be1eb2620e0e:40345 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0c1521ac61ff4c7e8e7d8a4ae2053686 2024-11-20T08:48:48,729 DEBUG [M:0;be1eb2620e0e:40345 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0c1521ac61ff4c7e8e7d8a4ae2053686 is 69, key is be1eb2620e0e,35761,1732092527797/rs:state/1732092527852/Put/seqid=0 2024-11-20T08:48:48,730 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0c1521ac61ff4c7e8e7d8a4ae2053686 2024-11-20T08:48:48,731 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0c1521ac61ff4c7e8e7d8a4ae2053686 2024-11-20T08:48:48,732 DEBUG [M:0;be1eb2620e0e:40345 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d81ad22834844e1ba3c34e2f22ca752 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d81ad22834844e1ba3c34e2f22ca752 2024-11-20T08:48:48,734 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d81ad22834844e1ba3c34e2f22ca752, entries=14, sequenceid=102, filesize=5.9 K 2024-11-20T08:48:48,734 DEBUG [M:0;be1eb2620e0e:40345 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2f549348be7b4dac9a6d0cb4a94290e4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2f549348be7b4dac9a6d0cb4a94290e4 2024-11-20T08:48:48,736 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2f549348be7b4dac9a6d0cb4a94290e4, entries=11, sequenceid=102, filesize=6.5 K 2024-11-20T08:48:48,736 DEBUG [M:0;be1eb2620e0e:40345 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0c1521ac61ff4c7e8e7d8a4ae2053686 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0c1521ac61ff4c7e8e7d8a4ae2053686 2024-11-20T08:48:48,738 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0c1521ac61ff4c7e8e7d8a4ae2053686 2024-11-20T08:48:48,739 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b27a107-e019-23c7-b6ba-cb5844f92c14/63dc0b38-5875-843f-f2d4-e078708af239/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0c1521ac61ff4c7e8e7d8a4ae2053686, entries=3, sequenceid=102, filesize=5.3 K 2024-11-20T08:48:48,740 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 80ms, sequenceid=102, compaction requested=false 2024-11-20T08:48:48,742 INFO [M:0;be1eb2620e0e:40345 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:48:48,742 DEBUG [M:0;be1eb2620e0e:40345 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732092528658Disabling compacts and flushes for region at 1732092528658Disabling writes for close at 1732092528659 (+1 ms)Obtaining lock to block concurrent updates at 1732092528659Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732092528659Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732092528660 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732092528660Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732092528660Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732092528683 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732092528683Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732092528685 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732092528705 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732092528705Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732092528708 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732092528728 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732092528728Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6de76343: reopening flushed file at 1732092528732 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e803896: reopening flushed file at 1732092528734 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e9d0242: reopening flushed file at 1732092528736 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 80ms, sequenceid=102, compaction requested=false at 1732092528740 (+4 ms)Writing region close event to WAL at 1732092528742 (+2 ms)Closed at 1732092528742 2024-11-20T08:48:48,743 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T08:48:48,743 INFO [M:0;be1eb2620e0e:40345 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T08:48:48,743 INFO [M:0;be1eb2620e0e:40345 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40345 2024-11-20T08:48:48,744 INFO [M:0;be1eb2620e0e:40345 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T08:48:48,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:48,745 INFO [RS:0;be1eb2620e0e:43559 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T08:48:48,745 INFO [RS:0;be1eb2620e0e:43559 {}] regionserver.HRegionServer(1031): Exiting; stopping=be1eb2620e0e,43559,1732092524519; zookeeper connection closed. 2024-11-20T08:48:48,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43559-0x100141e30850001, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:48,746 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a7b07b7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a7b07b7 2024-11-20T08:48:48,746 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T08:48:48,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:48,846 INFO [M:0;be1eb2620e0e:40345 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T08:48:48,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x100141e30850000, quorum=127.0.0.1:62105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T08:48:48,849 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T08:48:48,851 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T08:48:48,851 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:48:48,863 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=858 (was 924), ProcessCount=11 (was 11), AvailableMemoryMB=3668 (was 2925) - AvailableMemoryMB LEAK? -