2024-11-15 16:48:19,185 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-15 16:48:19,202 main DEBUG Took 0.014705 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 16:48:19,203 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 16:48:19,203 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 16:48:19,204 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 16:48:19,206 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,217 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 16:48:19,243 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,246 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,247 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,247 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,248 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,249 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,249 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,249 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,250 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,251 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,251 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,252 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,252 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,252 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,253 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,253 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,254 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,254 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,255 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,255 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,256 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:48:19,257 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,257 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 16:48:19,259 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:48:19,261 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 16:48:19,262 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 16:48:19,263 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 16:48:19,265 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 16:48:19,265 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 16:48:19,277 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 16:48:19,279 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 16:48:19,281 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 16:48:19,281 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 16:48:19,282 main DEBUG createAppenders(={Console}) 2024-11-15 16:48:19,282 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-15 16:48:19,283 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-15 16:48:19,283 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-15 16:48:19,284 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 16:48:19,284 main DEBUG OutputStream closed 2024-11-15 16:48:19,284 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 16:48:19,284 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 16:48:19,285 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-15 16:48:19,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 16:48:19,353 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 16:48:19,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 16:48:19,355 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 16:48:19,356 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 16:48:19,356 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 16:48:19,357 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 16:48:19,357 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 16:48:19,358 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 16:48:19,358 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 16:48:19,359 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 16:48:19,359 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 16:48:19,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 16:48:19,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 16:48:19,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 16:48:19,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 16:48:19,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 16:48:19,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 16:48:19,364 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 16:48:19,365 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-15 16:48:19,365 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 16:48:19,366 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-15T16:48:19,379 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-15 16:48:19,382 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 16:48:19,382 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T16:48:19,649 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a 2024-11-15T16:48:19,674 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1387, ProcessCount=11, AvailableMemoryMB=6511 2024-11-15T16:48:19,789 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T16:48:19,882 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:19,898 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/cluster_6a58f756-80da-555d-8b13-ce3e96517ee7, deleteOnExit=true 2024-11-15T16:48:19,986 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/cluster_6a58f756-80da-555d-8b13-ce3e96517ee7/zookeeper_0, clientPort=49978, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/cluster_6a58f756-80da-555d-8b13-ce3e96517ee7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/cluster_6a58f756-80da-555d-8b13-ce3e96517ee7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T16:48:19,999 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49978 2024-11-15T16:48:19,999 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:20,084 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 with version=8 2024-11-15T16:48:20,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/hbase-staging 2024-11-15T16:48:20,185 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T16:48:20,479 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:48:20,492 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:20,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:20,505 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:48:20,505 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:20,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:48:20,658 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T16:48:20,755 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T16:48:20,794 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T16:48:20,805 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T16:48:20,809 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:48:20,850 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20670 (auto-detected) 2024-11-15T16:48:20,852 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-15T16:48:20,879 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33999 2024-11-15T16:48:20,915 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33999 connecting to ZooKeeper ensemble=127.0.0.1:49978 2024-11-15T16:48:20,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:339990x0, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:48:20,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33999-0x101acd535610000 connected 2024-11-15T16:48:20,987 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:20,988 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:20,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:21,002 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913, hbase.cluster.distributed=false 2024-11-15T16:48:21,030 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:48:21,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33999 2024-11-15T16:48:21,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33999 2024-11-15T16:48:21,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33999 2024-11-15T16:48:21,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33999 2024-11-15T16:48:21,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33999 2024-11-15T16:48:21,136 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:48:21,138 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:21,138 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:21,138 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:48:21,139 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:21,139 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:48:21,142 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T16:48:21,144 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:48:21,145 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35489 2024-11-15T16:48:21,147 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35489 connecting to ZooKeeper ensemble=127.0.0.1:49978 2024-11-15T16:48:21,148 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:21,149 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:21,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354890x0, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:48:21,154 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:354890x0, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:21,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35489-0x101acd535610001 connected 2024-11-15T16:48:21,160 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T16:48:21,174 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.cache 2024-11-15T16:48:21,181 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T16:48:21,183 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T16:48:21,185 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T16:48:21,187 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T16:48:21,187 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-15T16:48:21,194 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T16:48:21,199 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T16:48:21,199 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.persistence is complete. 2024-11-15T16:48:21,203 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T16:48:21,206 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T16:48:21,212 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:48:21,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35489 2024-11-15T16:48:21,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35489 2024-11-15T16:48:21,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35489 2024-11-15T16:48:21,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35489 2024-11-15T16:48:21,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35489 2024-11-15T16:48:21,254 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:48:21,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:21,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:21,255 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:48:21,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:21,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:48:21,256 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T16:48:21,257 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:48:21,258 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37651 2024-11-15T16:48:21,261 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37651 connecting to ZooKeeper ensemble=127.0.0.1:49978 2024-11-15T16:48:21,262 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:21,263 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:21,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376510x0, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:48:21,270 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:376510x0, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:21,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37651-0x101acd535610002 connected 2024-11-15T16:48:21,271 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T16:48:21,273 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.cache 2024-11-15T16:48:21,273 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T16:48:21,273 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T16:48:21,274 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T16:48:21,276 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T16:48:21,276 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-15T16:48:21,277 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T16:48:21,277 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.persistence is complete. 2024-11-15T16:48:21,280 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T16:48:21,280 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T16:48:21,282 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T16:48:21,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:48:21,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37651 2024-11-15T16:48:21,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37651 2024-11-15T16:48:21,293 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37651 2024-11-15T16:48:21,295 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37651 2024-11-15T16:48:21,296 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37651 2024-11-15T16:48:21,323 DEBUG [M:0;a9bde5b3a1b5:33999 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a9bde5b3a1b5:33999 2024-11-15T16:48:21,325 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:21,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:21,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:21,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:21,335 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:21,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T16:48:21,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T16:48:21,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:21,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:21,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:21,363 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T16:48:21,365 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a9bde5b3a1b5,33999,1731689300276 from backup master directory 2024-11-15T16:48:21,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:21,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:21,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:21,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:21,373 WARN [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:48:21,373 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:21,375 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T16:48:21,377 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T16:48:21,386 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/hbase.id] with ID: 45dca503-7862-4e87-a03e-e2a19115df07 2024-11-15T16:48:21,387 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/.tmp/hbase.id 2024-11-15T16:48:21,394 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/hbase.id] 2024-11-15T16:48:21,435 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:21,440 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T16:48:21,461 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-15T16:48:21,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:21,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:21,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:21,493 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:48:21,498 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T16:48:21,552 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:48:21,559 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:21,634 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store 2024-11-15T16:48:21,661 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T16:48:21,665 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:21,667 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T16:48:21,667 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:21,667 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:21,670 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T16:48:21,670 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:21,670 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:21,671 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731689301667Disabling compacts and flushes for region at 1731689301667Disabling writes for close at 1731689301670 (+3 ms)Writing region close event to WAL at 1731689301670Closed at 1731689301670 2024-11-15T16:48:21,672 WARN [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/.initializing 2024-11-15T16:48:21,672 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/WALs/a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:21,688 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:48:21,703 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C33999%2C1731689300276, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/WALs/a9bde5b3a1b5,33999,1731689300276, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/oldWALs, maxLogs=10 2024-11-15T16:48:21,749 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/WALs/a9bde5b3a1b5,33999,1731689300276/a9bde5b3a1b5%2C33999%2C1731689300276.1731689301709 2024-11-15T16:48:21,750 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:21,751 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:21,752 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:21,756 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,758 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T16:48:21,846 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:21,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:21,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T16:48:21,854 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:21,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:21,855 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T16:48:21,859 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:21,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:21,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,864 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T16:48:21,864 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:21,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:21,865 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,869 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,869 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,872 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,873 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,876 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:48:21,879 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:21,894 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:48:21,895 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66523543, jitterRate=-0.008721962571144104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:48:21,903 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731689301774Initializing all the Stores at 1731689301776 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689301777 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689301779 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689301779Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689301779Cleaning up temporary data from old regions at 1731689301873 (+94 ms)Region opened successfully at 1731689301903 (+30 ms) 2024-11-15T16:48:21,905 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T16:48:21,946 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f9dad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:48:21,985 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T16:48:21,997 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T16:48:21,998 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T16:48:22,001 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T16:48:22,001 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T16:48:22,007 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T16:48:22,007 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T16:48:22,033 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T16:48:22,043 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T16:48:22,047 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T16:48:22,050 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T16:48:22,051 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T16:48:22,054 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T16:48:22,056 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T16:48:22,061 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T16:48:22,063 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T16:48:22,065 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T16:48:22,066 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T16:48:22,087 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T16:48:22,089 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T16:48:22,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:22,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:22,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:22,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,107 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a9bde5b3a1b5,33999,1731689300276, sessionid=0x101acd535610000, setting cluster-up flag (Was=false) 2024-11-15T16:48:22,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,129 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T16:48:22,131 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:22,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,144 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T16:48:22,146 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:22,151 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T16:48:22,204 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(746): ClusterId : 45dca503-7862-4e87-a03e-e2a19115df07 2024-11-15T16:48:22,204 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(746): ClusterId : 45dca503-7862-4e87-a03e-e2a19115df07 2024-11-15T16:48:22,207 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T16:48:22,207 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T16:48:22,214 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T16:48:22,214 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T16:48:22,214 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T16:48:22,214 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T16:48:22,218 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T16:48:22,219 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a118f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:48:22,225 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T16:48:22,225 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7386809a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:48:22,230 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T16:48:22,235 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a9bde5b3a1b5:35489 2024-11-15T16:48:22,239 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T16:48:22,239 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T16:48:22,239 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T16:48:22,243 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,33999,1731689300276 with port=35489, startcode=1731689301089 2024-11-15T16:48:22,245 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T16:48:22,246 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a9bde5b3a1b5:37651 2024-11-15T16:48:22,246 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T16:48:22,246 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T16:48:22,247 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T16:48:22,248 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,33999,1731689300276 with port=37651, startcode=1731689301253 2024-11-15T16:48:22,256 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T16:48:22,259 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T16:48:22,259 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T16:48:22,263 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a9bde5b3a1b5,33999,1731689300276 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T16:48:22,270 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:48:22,270 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:48:22,271 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:48:22,271 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:48:22,271 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a9bde5b3a1b5:0, corePoolSize=10, maxPoolSize=10 2024-11-15T16:48:22,271 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,272 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:48:22,272 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,274 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731689332274 2024-11-15T16:48:22,276 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:48:22,277 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T16:48:22,277 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T16:48:22,280 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T16:48:22,283 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:22,283 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T16:48:22,284 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T16:48:22,284 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T16:48:22,285 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T16:48:22,285 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T16:48:22,287 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,291 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T16:48:22,293 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T16:48:22,293 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T16:48:22,304 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T16:48:22,304 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T16:48:22,307 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689302306,5,FailOnTimeoutGroup] 2024-11-15T16:48:22,307 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689302307,5,FailOnTimeoutGroup] 2024-11-15T16:48:22,308 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,308 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T16:48:22,309 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,309 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,321 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T16:48:22,322 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 2024-11-15T16:48:22,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:22,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-15T16:48:22,352 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53399, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T16:48:22,353 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53627, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T16:48:22,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:48:22,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:22,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:22,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-15T16:48:22,361 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:48:22,361 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:22,361 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33999 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:22,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:22,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-15T16:48:22,364 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33999 {}] master.ServerManager(517): Registering regionserver=a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:22,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:48:22,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:22,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:22,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-15T16:48:22,378 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33999 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:22,379 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33999 {}] master.ServerManager(517): Registering regionserver=a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:22,384 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 2024-11-15T16:48:22,384 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T16:48:22,384 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T16:48:22,385 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 2024-11-15T16:48:22,385 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T16:48:22,385 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T16:48:22,389 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:48:22,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:22,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:22,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:48:22,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:22,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:22,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:48:22,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:48:22,393 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:48:22,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:48:22,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:48:22,396 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] zookeeper.ZKUtil(111): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:22,396 WARN [RS:1;a9bde5b3a1b5:37651 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:48:22,396 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] zookeeper.ZKUtil(111): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:22,396 WARN [RS:0;a9bde5b3a1b5:35489 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:48:22,397 INFO [RS:1;a9bde5b3a1b5:37651 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:22,397 INFO [RS:0;a9bde5b3a1b5:35489 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:22,397 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:22,397 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:22,402 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a9bde5b3a1b5,37651,1731689301253] 2024-11-15T16:48:22,402 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a9bde5b3a1b5,35489,1731689301089] 2024-11-15T16:48:22,413 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:48:22,414 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72298257, jitterRate=0.07732798159122467}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:48:22,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731689302344Initializing all the Stores at 1731689302345 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689302345Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689302346 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689302346Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689302346Cleaning up temporary data from old regions at 1731689302393 (+47 ms)Region opened successfully at 1731689302418 (+25 ms) 2024-11-15T16:48:22,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T16:48:22,419 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T16:48:22,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T16:48:22,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T16:48:22,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T16:48:22,425 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T16:48:22,425 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731689302419Disabling compacts and flushes for region at 1731689302419Disabling writes for close at 1731689302419Writing region close event to WAL at 1731689302424 (+5 ms)Closed at 1731689302425 (+1 ms) 2024-11-15T16:48:22,427 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:48:22,427 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T16:48:22,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T16:48:22,441 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T16:48:22,441 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T16:48:22,447 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:48:22,450 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T16:48:22,462 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T16:48:22,465 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T16:48:22,469 INFO [RS:0;a9bde5b3a1b5:35489 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:48:22,469 INFO [RS:1;a9bde5b3a1b5:37651 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:48:22,469 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,469 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,473 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T16:48:22,475 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T16:48:22,479 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T16:48:22,480 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T16:48:22,481 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,481 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,482 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,482 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,482 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,482 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,482 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,482 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,483 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,483 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,483 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,483 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,483 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:48:22,483 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:48:22,484 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,484 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,484 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,484 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,484 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,484 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,484 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,484 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,485 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,485 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,485 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,485 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:22,485 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:22,485 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:22,485 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:22,485 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:22,495 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,496 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,496 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,496 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,496 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,497 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,37651,1731689301253-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:48:22,498 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,498 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,498 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,498 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,498 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,499 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,35489,1731689301089-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:48:22,521 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T16:48:22,521 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T16:48:22,524 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,35489,1731689301089-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,524 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,37651,1731689301253-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,524 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,524 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,524 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.Replication(171): a9bde5b3a1b5,35489,1731689301089 started 2024-11-15T16:48:22,524 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.Replication(171): a9bde5b3a1b5,37651,1731689301253 started 2024-11-15T16:48:22,553 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,553 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:22,554 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(1482): Serving as a9bde5b3a1b5,37651,1731689301253, RpcServer on a9bde5b3a1b5/172.17.0.2:37651, sessionid=0x101acd535610002 2024-11-15T16:48:22,554 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(1482): Serving as a9bde5b3a1b5,35489,1731689301089, RpcServer on a9bde5b3a1b5/172.17.0.2:35489, sessionid=0x101acd535610001 2024-11-15T16:48:22,555 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T16:48:22,555 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T16:48:22,555 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:22,555 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:22,555 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,35489,1731689301089' 2024-11-15T16:48:22,555 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,37651,1731689301253' 2024-11-15T16:48:22,555 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T16:48:22,555 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T16:48:22,557 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T16:48:22,557 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T16:48:22,558 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T16:48:22,558 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T16:48:22,558 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T16:48:22,558 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T16:48:22,558 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:22,558 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:22,558 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,37651,1731689301253' 2024-11-15T16:48:22,558 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,35489,1731689301089' 2024-11-15T16:48:22,558 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T16:48:22,558 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T16:48:22,559 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T16:48:22,559 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T16:48:22,559 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T16:48:22,560 INFO [RS:1;a9bde5b3a1b5:37651 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T16:48:22,560 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T16:48:22,560 INFO [RS:1;a9bde5b3a1b5:37651 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T16:48:22,560 INFO [RS:0;a9bde5b3a1b5:35489 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T16:48:22,560 INFO [RS:0;a9bde5b3a1b5:35489 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T16:48:22,601 WARN [a9bde5b3a1b5:33999 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-15T16:48:22,666 INFO [RS:1;a9bde5b3a1b5:37651 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:48:22,666 INFO [RS:0;a9bde5b3a1b5:35489 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:48:22,667 INFO [RS:1;a9bde5b3a1b5:37651 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a9bde5b3a1b5%2C37651%2C1731689301253, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,37651,1731689301253, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs, maxLogs=55 2024-11-15T16:48:22,667 INFO [RS:0;a9bde5b3a1b5:35489 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a9bde5b3a1b5%2C35489%2C1731689301089, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,35489,1731689301089, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs, maxLogs=55 2024-11-15T16:48:22,694 INFO [RS:1;a9bde5b3a1b5:37651 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,37651,1731689301253/a9bde5b3a1b5%2C37651%2C1731689301253.1731689302671 2024-11-15T16:48:22,695 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:22,695 INFO [RS:0;a9bde5b3a1b5:35489 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,35489,1731689301089/a9bde5b3a1b5%2C35489%2C1731689301089.1731689302672 2024-11-15T16:48:22,695 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:22,854 DEBUG [a9bde5b3a1b5:33999 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-15T16:48:22,865 DEBUG [a9bde5b3a1b5:33999 {}] balancer.BalancerClusterState(204): Hosts are {a9bde5b3a1b5=0} racks are {/default-rack=0} 2024-11-15T16:48:22,869 DEBUG [a9bde5b3a1b5:33999 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T16:48:22,869 DEBUG [a9bde5b3a1b5:33999 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T16:48:22,869 DEBUG [a9bde5b3a1b5:33999 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T16:48:22,869 DEBUG [a9bde5b3a1b5:33999 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T16:48:22,869 INFO [a9bde5b3a1b5:33999 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T16:48:22,869 INFO [a9bde5b3a1b5:33999 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T16:48:22,869 DEBUG [a9bde5b3a1b5:33999 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T16:48:22,878 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:22,884 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,37651,1731689301253, state=OPENING 2024-11-15T16:48:22,889 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T16:48:22,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:22,892 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:22,892 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:22,892 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:22,893 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:48:22,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,37651,1731689301253}] 2024-11-15T16:48:23,070 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:48:23,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56563, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:48:23,082 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T16:48:23,082 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:23,083 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T16:48:23,084 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a9bde5b3a1b5%2C37651%2C1731689301253.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,37651,1731689301253, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs, maxLogs=55 2024-11-15T16:48:23,106 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,37651,1731689301253/a9bde5b3a1b5%2C37651%2C1731689301253.meta.1731689303086.meta 2024-11-15T16:48:23,106 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:23,107 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:23,109 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T16:48:23,111 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T16:48:23,117 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T16:48:23,121 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T16:48:23,122 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:23,122 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T16:48:23,123 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T16:48:23,125 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-15T16:48:23,126 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:48:23,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:23,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:23,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-15T16:48:23,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:48:23,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:23,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:23,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-15T16:48:23,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:48:23,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:23,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:23,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-15T16:48:23,131 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:48:23,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:23,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:23,131 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:48:23,132 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:23,133 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:23,133 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:48:23,133 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:48:23,134 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:48:23,135 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:48:23,136 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61415071, jitterRate=-0.08484412729740143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:48:23,136 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T16:48:23,139 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731689303123Writing region info on filesystem at 1731689303124 (+1 ms)Initializing all the Stores at 1731689303124Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689303124Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689303125 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689303125Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689303125Cleaning up temporary data from old regions at 1731689303133 (+8 ms)Running coprocessor post-open hooks at 1731689303136 (+3 ms)Region opened successfully at 1731689303138 (+2 ms) 2024-11-15T16:48:23,146 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731689303060 2024-11-15T16:48:23,157 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T16:48:23,157 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T16:48:23,158 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:23,159 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,37651,1731689301253, state=OPEN 2024-11-15T16:48:23,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:23,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:23,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:23,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:23,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:23,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:23,163 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:23,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T16:48:23,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,37651,1731689301253 in 268 msec 2024-11-15T16:48:23,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T16:48:23,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 731 msec 2024-11-15T16:48:23,170 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:48:23,170 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T16:48:23,191 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:48:23,193 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,37651,1731689301253, seqNum=-1] 2024-11-15T16:48:23,213 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:48:23,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33435, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:48:23,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0540 sec 2024-11-15T16:48:23,234 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731689303234, completionTime=-1 2024-11-15T16:48:23,238 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-15T16:48:23,238 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T16:48:23,268 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T16:48:23,269 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731689363269 2024-11-15T16:48:23,269 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731689423269 2024-11-15T16:48:23,269 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-15T16:48:23,271 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-15T16:48:23,278 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,33999,1731689300276-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:23,278 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,33999,1731689300276-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:23,278 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,33999,1731689300276-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:23,280 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a9bde5b3a1b5:33999, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:23,280 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:23,281 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:23,289 DEBUG [master/a9bde5b3a1b5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T16:48:23,314 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.940sec 2024-11-15T16:48:23,316 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T16:48:23,317 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T16:48:23,319 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T16:48:23,319 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T16:48:23,320 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T16:48:23,321 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,33999,1731689300276-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:48:23,321 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,33999,1731689300276-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T16:48:23,323 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T16:48:23,324 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T16:48:23,324 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,33999,1731689300276-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:23,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4248d59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:48:23,416 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T16:48:23,416 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T16:48:23,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,33999,-1 for getting cluster id 2024-11-15T16:48:23,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:48:23,454 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45dca503-7862-4e87-a03e-e2a19115df07' 2024-11-15T16:48:23,457 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:48:23,457 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45dca503-7862-4e87-a03e-e2a19115df07" 2024-11-15T16:48:23,459 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd1daa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:48:23,460 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,33999,-1] 2024-11-15T16:48:23,462 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:48:23,463 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:23,465 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:48:23,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f82a59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:48:23,468 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:48:23,475 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,37651,1731689301253, seqNum=-1] 2024-11-15T16:48:23,475 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:48:23,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:48:23,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:23,501 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T16:48:23,506 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:23,509 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1021df04 2024-11-15T16:48:23,510 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T16:48:23,512 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T16:48:23,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731689303497', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:48:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731689303497 2024-11-15T16:48:23,530 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731689303497 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T16:48:23,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731689303497" procId is: 4 2024-11-15T16:48:23,533 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:23,534 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731689303497 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T16:48:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:48:23,553 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731689303497-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b6a5bccf03f461ca7448a06c3e9e509, NAME => 'testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731689303497', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 2024-11-15T16:48:23,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731689303497-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:23,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731689303497-pool-0 {}] regionserver.HRegion(1722): Closing 8b6a5bccf03f461ca7448a06c3e9e509, disabling compactions & flushes 2024-11-15T16:48:23,561 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731689303497-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:23,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731689303497-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:23,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731689303497-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. after waiting 0 ms 2024-11-15T16:48:23,561 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731689303497-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:23,561 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731689303497-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:23,562 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731689303497-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b6a5bccf03f461ca7448a06c3e9e509: Waiting for close lock at 1731689303561Disabling compacts and flushes for region at 1731689303561Disabling writes for close at 1731689303561Writing region close event to WAL at 1731689303561Closed at 1731689303561 2024-11-15T16:48:23,563 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731689303497 execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T16:48:23,570 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731689303563"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731689303563"}]},"ts":"1731689303563"} 2024-11-15T16:48:23,576 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T16:48:23,578 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731689303497 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T16:48:23,582 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731689303497","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689303578"}]},"ts":"1731689303578"} 2024-11-15T16:48:23,587 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731689303497, state=ENABLING in hbase:meta 2024-11-15T16:48:23,588 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a9bde5b3a1b5=0} racks are {/default-rack=0} 2024-11-15T16:48:23,588 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T16:48:23,589 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T16:48:23,589 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T16:48:23,589 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T16:48:23,589 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T16:48:23,589 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T16:48:23,589 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T16:48:23,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, ASSIGN}] 2024-11-15T16:48:23,593 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, ASSIGN 2024-11-15T16:48:23,595 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, ASSIGN; state=OFFLINE, location=a9bde5b3a1b5,37651,1731689301253; forceNewPlan=false, retain=false 2024-11-15T16:48:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:48:23,748 INFO [a9bde5b3a1b5:33999 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T16:48:23,749 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8b6a5bccf03f461ca7448a06c3e9e509, regionState=OPENING, regionLocation=a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:23,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, ASSIGN because future has completed 2024-11-15T16:48:23,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b6a5bccf03f461ca7448a06c3e9e509, server=a9bde5b3a1b5,37651,1731689301253}] 2024-11-15T16:48:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:48:23,912 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:23,912 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b6a5bccf03f461ca7448a06c3e9e509, NAME => 'testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:23,913 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731689303497 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,913 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:23,913 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,913 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,914 INFO [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,919 INFO [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b6a5bccf03f461ca7448a06c3e9e509 columnFamilyName family 2024-11-15T16:48:23,920 DEBUG [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:23,920 INFO [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] regionserver.HStore(327): Store=8b6a5bccf03f461ca7448a06c3e9e509/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:23,920 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,921 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,922 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,922 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,922 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,923 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,929 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:48:23,929 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8b6a5bccf03f461ca7448a06c3e9e509; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59801497, jitterRate=-0.10888825356960297}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:48:23,930 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:23,930 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8b6a5bccf03f461ca7448a06c3e9e509: Running coprocessor pre-open hook at 1731689303913Writing region info on filesystem at 1731689303913Initializing all the Stores at 1731689303914 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689303914Cleaning up temporary data from old regions at 1731689303922 (+8 ms)Running coprocessor post-open hooks at 1731689303930 (+8 ms)Region opened successfully at 1731689303930 2024-11-15T16:48:23,932 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509., pid=6, masterSystemTime=1731689303908 2024-11-15T16:48:23,935 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:23,935 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:23,935 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8b6a5bccf03f461ca7448a06c3e9e509, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:23,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b6a5bccf03f461ca7448a06c3e9e509, server=a9bde5b3a1b5,37651,1731689301253 because future has completed 2024-11-15T16:48:23,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T16:48:23,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b6a5bccf03f461ca7448a06c3e9e509, server=a9bde5b3a1b5,37651,1731689301253 in 185 msec 2024-11-15T16:48:23,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T16:48:23,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, ASSIGN in 351 msec 2024-11-15T16:48:23,945 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731689303497 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T16:48:23,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731689303497","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689303945"}]},"ts":"1731689303945"} 2024-11-15T16:48:23,948 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731689303497, state=ENABLED in hbase:meta 2024-11-15T16:48:23,949 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731689303497 execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T16:48:23,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731689303497 in 426 msec 2024-11-15T16:48:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:48:24,173 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731689303497 completed 2024-11-15T16:48:24,173 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731689303497 get assigned. Timeout = 60000ms 2024-11-15T16:48:24,177 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T16:48:24,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731689303497 assigned to meta. Checking AM states. 2024-11-15T16:48:24,185 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T16:48:24,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731689303497 assigned. 2024-11-15T16:48:24,196 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731689303497', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509., hostname=a9bde5b3a1b5,37651,1731689301253, seqNum=2] 2024-11-15T16:48:24,209 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8b6a5bccf03f461ca7448a06c3e9e509 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-15T16:48:24,236 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 26c1092da2ba400fbabc94cb199754dc 2024-11-15T16:48:24,290 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/.tmp/family/26c1092da2ba400fbabc94cb199754dc is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-15T16:48:24,297 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/.tmp/family/26c1092da2ba400fbabc94cb199754dc 2024-11-15T16:48:24,339 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/.tmp/family/26c1092da2ba400fbabc94cb199754dc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc 2024-11-15T16:48:24,340 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc, delay=922 ms 2024-11-15T16:48:24,342 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc, entries=2, sequenceid=6, filesize=4.9 K 2024-11-15T16:48:24,348 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 8b6a5bccf03f461ca7448a06c3e9e509 in 138ms, sequenceid=6, compaction requested=false 2024-11-15T16:48:24,349 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731689303497' 2024-11-15T16:48:24,351 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8b6a5bccf03f461ca7448a06c3e9e509: 2024-11-15T16:48:25,265 DEBUG [hfile-prefetch-1731689304340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 26c1092da2ba400fbabc94cb199754dc_0. 2024-11-15T16:48:25,269 DEBUG [hfile-prefetch-1731689304340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 26c1092da2ba400fbabc94cb199754dc_117. 2024-11-15T16:48:25,270 DEBUG [hfile-prefetch-1731689304340 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc, with total blocks 2, and data blocks 1 2024-11-15T16:48:25,270 DEBUG [hfile-prefetch-1731689304340 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-15T16:48:25,271 DEBUG [hfile-prefetch-1731689304340 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-15T16:48:25,271 DEBUG [hfile-prefetch-1731689304340 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc. Will sleep 100ms and try the verification again. 2024-11-15T16:48:25,278 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-15T16:48:25,278 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-15T16:48:25,278 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:25,278 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:25,288 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-15T16:48:25,289 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-15T16:48:25,289 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:25,289 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:25,299 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T16:48:25,299 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731689305289 2024-11-15T16:48:25,299 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-15T16:48:25,302 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-15T16:48:25,302 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731689305278 2024-11-15T16:48:25,302 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-15T16:48:25,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8b6a5bccf03f461ca7448a06c3e9e509, source=a9bde5b3a1b5,37651,1731689301253, destination=a9bde5b3a1b5,35489,1731689301089, warming up region on a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:48:25,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8b6a5bccf03f461ca7448a06c3e9e509, source=a9bde5b3a1b5,37651,1731689301253, destination=a9bde5b3a1b5,35489,1731689301089, running balancer 2024-11-15T16:48:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, REOPEN/MOVE 2024-11-15T16:48:25,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55623, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:48:25,369 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, REOPEN/MOVE 2024-11-15T16:48:25,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8b6a5bccf03f461ca7448a06c3e9e509, NAME => 'testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:25,371 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8b6a5bccf03f461ca7448a06c3e9e509, regionState=CLOSING, regionLocation=a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:25,371 DEBUG [hfile-prefetch-1731689304340 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc, with total blocks 2, and data blocks 1 2024-11-15T16:48:25,371 DEBUG [hfile-prefetch-1731689304340 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-15T16:48:25,371 DEBUG [hfile-prefetch-1731689304340 {}] bucket.BucketCache(2293): found block for file 26c1092da2ba400fbabc94cb199754dc in the backing map. Acquiring read lock for offset 0 2024-11-15T16:48:25,371 INFO [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,372 DEBUG [hfile-prefetch-1731689304340 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc has now been fully cached. 2024-11-15T16:48:25,372 DEBUG [hfile-prefetch-1731689304340 {}] hfile.PrefetchExecutor(121): Prefetch completed for 26c1092da2ba400fbabc94cb199754dc 2024-11-15T16:48:25,373 INFO [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b6a5bccf03f461ca7448a06c3e9e509 columnFamilyName family 2024-11-15T16:48:25,373 DEBUG [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:25,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, REOPEN/MOVE because future has completed 2024-11-15T16:48:25,378 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-15T16:48:25,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b6a5bccf03f461ca7448a06c3e9e509, server=a9bde5b3a1b5,37651,1731689301253}] 2024-11-15T16:48:25,380 DEBUG [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc 2024-11-15T16:48:25,380 INFO [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] regionserver.HStore(327): Store=8b6a5bccf03f461ca7448a06c3e9e509/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.HRegion(1722): Closing 8b6a5bccf03f461ca7448a06c3e9e509, disabling compactions & flushes 2024-11-15T16:48:25,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. after waiting 0 ms 2024-11-15T16:48:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35489 {}] regionserver.HRegion(1676): Region close journal for 8b6a5bccf03f461ca7448a06c3e9e509: Waiting for close lock at 1731689305380Disabling compacts and flushes for region at 1731689305380Disabling writes for close at 1731689305381 (+1 ms)Writing region close event to WAL at 1731689305382 (+1 ms)Closed at 1731689305382 2024-11-15T16:48:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33999 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-15T16:48:25,537 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,538 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-15T16:48:25,538 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 8b6a5bccf03f461ca7448a06c3e9e509, disabling compactions & flushes 2024-11-15T16:48:25,538 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,539 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,539 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. after waiting 0 ms 2024-11-15T16:48:25,539 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,541 DEBUG [StoreFileCloser-8b6a5bccf03f461ca7448a06c3e9e509-family-1 {}] bucket.BucketCache(768): Removing file 26c1092da2ba400fbabc94cb199754dc for region 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,541 DEBUG [StoreFileCloser-8b6a5bccf03f461ca7448a06c3e9e509-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 26c1092da2ba400fbabc94cb199754dc, starting offset: 0, end offset: 9223372036854775807 2024-11-15T16:48:25,543 DEBUG [StoreFileCloser-8b6a5bccf03f461ca7448a06c3e9e509-family-1 {}] bucket.BucketCache(856): removed key 26c1092da2ba400fbabc94cb199754dc_0 from back map with offset lock 2094080 in the evict process 2024-11-15T16:48:25,548 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-15T16:48:25,550 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,550 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 8b6a5bccf03f461ca7448a06c3e9e509: Waiting for close lock at 1731689305538Running coprocessor pre-close hooks at 1731689305538Disabling compacts and flushes for region at 1731689305538Disabling writes for close at 1731689305539 (+1 ms)Writing region close event to WAL at 1731689305543 (+4 ms)Running coprocessor post-close hooks at 1731689305548 (+5 ms)Closed at 1731689305550 (+2 ms) 2024-11-15T16:48:25,550 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 8b6a5bccf03f461ca7448a06c3e9e509 move to a9bde5b3a1b5,35489,1731689301089 record at close sequenceid=6 2024-11-15T16:48:25,553 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,554 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8b6a5bccf03f461ca7448a06c3e9e509, regionState=CLOSED 2024-11-15T16:48:25,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b6a5bccf03f461ca7448a06c3e9e509, server=a9bde5b3a1b5,37651,1731689301253 because future has completed 2024-11-15T16:48:25,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T16:48:25,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 8b6a5bccf03f461ca7448a06c3e9e509, server=a9bde5b3a1b5,37651,1731689301253 in 180 msec 2024-11-15T16:48:25,560 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, REOPEN/MOVE; state=CLOSED, location=a9bde5b3a1b5,35489,1731689301089; forceNewPlan=false, retain=false 2024-11-15T16:48:25,599 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-15T16:48:25,599 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-15T16:48:25,600 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:25,600 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:25,603 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-15T16:48:25,603 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-15T16:48:25,603 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:25,603 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:25,603 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T16:48:25,603 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731689305599 2024-11-15T16:48:25,603 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-15T16:48:25,606 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T16:48:25,606 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731689305603 2024-11-15T16:48:25,606 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-15T16:48:25,711 INFO [a9bde5b3a1b5:33999 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T16:48:25,712 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8b6a5bccf03f461ca7448a06c3e9e509, regionState=OPENING, regionLocation=a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:25,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, REOPEN/MOVE because future has completed 2024-11-15T16:48:25,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b6a5bccf03f461ca7448a06c3e9e509, server=a9bde5b3a1b5,35489,1731689301089}] 2024-11-15T16:48:25,872 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,873 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b6a5bccf03f461ca7448a06c3e9e509, NAME => 'testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:25,873 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731689303497 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,873 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:25,873 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,873 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,874 INFO [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,875 INFO [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b6a5bccf03f461ca7448a06c3e9e509 columnFamilyName family 2024-11-15T16:48:25,876 DEBUG [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:25,877 DEBUG [StoreFileOpener-8b6a5bccf03f461ca7448a06c3e9e509-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc, delay=926 ms 2024-11-15T16:48:25,879 DEBUG [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc 2024-11-15T16:48:25,879 INFO [StoreOpener-8b6a5bccf03f461ca7448a06c3e9e509-1 {}] regionserver.HStore(327): Store=8b6a5bccf03f461ca7448a06c3e9e509/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:25,879 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,880 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,881 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,881 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,881 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,886 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,887 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 8b6a5bccf03f461ca7448a06c3e9e509; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60387467, jitterRate=-0.10015662014484406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:48:25,887 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,888 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8b6a5bccf03f461ca7448a06c3e9e509: Running coprocessor pre-open hook at 1731689305874Writing region info on filesystem at 1731689305874Initializing all the Stores at 1731689305874Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689305874Cleaning up temporary data from old regions at 1731689305881 (+7 ms)Running coprocessor post-open hooks at 1731689305887 (+6 ms)Region opened successfully at 1731689305888 (+1 ms) 2024-11-15T16:48:25,889 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509., pid=9, masterSystemTime=1731689305869 2024-11-15T16:48:25,892 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,892 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,893 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8b6a5bccf03f461ca7448a06c3e9e509, regionState=OPEN, openSeqNum=10, regionLocation=a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:25,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b6a5bccf03f461ca7448a06c3e9e509, server=a9bde5b3a1b5,35489,1731689301089 because future has completed 2024-11-15T16:48:25,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-15T16:48:25,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b6a5bccf03f461ca7448a06c3e9e509, server=a9bde5b3a1b5,35489,1731689301089 in 181 msec 2024-11-15T16:48:25,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731689303497, region=8b6a5bccf03f461ca7448a06c3e9e509, REOPEN/MOVE in 532 msec 2024-11-15T16:48:25,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T16:48:25,922 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:48:25,922 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:48:25,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:25,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:25,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:48:25,927 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T16:48:25,927 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-15T16:48:25,928 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a9bde5b3a1b5,33999,1731689300276 2024-11-15T16:48:25,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:25,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:25,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:25,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:25,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:25,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:25,931 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T16:48:25,931 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:48:25,932 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:48:25,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:25,932 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:25,932 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:25,932 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a9bde5b3a1b5,35489,1731689301089' ***** 2024-11-15T16:48:25,932 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T16:48:25,933 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a9bde5b3a1b5,37651,1731689301253' ***** 2024-11-15T16:48:25,933 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T16:48:25,933 INFO [RS:0;a9bde5b3a1b5:35489 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T16:48:25,933 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:25,933 INFO [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T16:48:25,933 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T16:48:25,933 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T16:48:25,934 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T16:48:25,934 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T16:48:25,934 INFO [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.persistence 2024-11-15T16:48:25,934 INFO [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T16:48:25,934 INFO [RS:0;a9bde5b3a1b5:35489 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/bucket.persistence 2024-11-15T16:48:25,934 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T16:48:25,934 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T16:48:25,934 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T16:48:25,934 INFO [RS:0;a9bde5b3a1b5:35489 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T16:48:25,935 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T16:48:25,935 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] bucket.BucketCache(1487): Thread RS:0;a9bde5b3a1b5:35489 started persisting bucket cache to file 2024-11-15T16:48:25,935 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1487): Thread RS:1;a9bde5b3a1b5:37651 started persisting bucket cache to file 2024-11-15T16:48:25,935 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:25,935 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:25,935 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:25,935 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:25,939 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T16:48:25,939 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731689305935 2024-11-15T16:48:25,939 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] bucket.BucketCache(1509): Thread RS:0;a9bde5b3a1b5:35489 finished persisting bucket cache to file, renaming 2024-11-15T16:48:25,939 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T16:48:25,939 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731689305935 2024-11-15T16:48:25,939 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1509): Thread RS:1;a9bde5b3a1b5:37651 finished persisting bucket cache to file, renaming 2024-11-15T16:48:25,939 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T16:48:25,939 WARN [RS:1;a9bde5b3a1b5:37651 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-15T16:48:25,939 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T16:48:25,940 INFO [RS:0;a9bde5b3a1b5:35489 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T16:48:25,940 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T16:48:25,940 INFO [RS:1;a9bde5b3a1b5:37651 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T16:48:25,940 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T16:48:25,940 INFO [RS:0;a9bde5b3a1b5:35489 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T16:48:25,940 INFO [RS:1;a9bde5b3a1b5:37651 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T16:48:25,940 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(3091): Received CLOSE for 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,940 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(959): stopping server a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:25,940 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:48:25,940 INFO [RS:1;a9bde5b3a1b5:37651 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a9bde5b3a1b5:37651. 2024-11-15T16:48:25,940 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:48:25,940 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:25,941 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(959): stopping server a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:25,941 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:48:25,941 INFO [RS:0;a9bde5b3a1b5:35489 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a9bde5b3a1b5:35489. 2024-11-15T16:48:25,941 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T16:48:25,941 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:48:25,941 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T16:48:25,941 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:25,941 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T16:48:25,941 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T16:48:25,941 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T16:48:25,941 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T16:48:25,941 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(1325): Online Regions={8b6a5bccf03f461ca7448a06c3e9e509=testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509.} 2024-11-15T16:48:25,941 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T16:48:25,942 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T16:48:25,942 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8b6a5bccf03f461ca7448a06c3e9e509, disabling compactions & flushes 2024-11-15T16:48:25,942 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(1351): Waiting on 8b6a5bccf03f461ca7448a06c3e9e509 2024-11-15T16:48:25,942 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T16:48:25,942 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,942 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,942 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T16:48:25,942 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. after waiting 0 ms 2024-11-15T16:48:25,942 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,942 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T16:48:25,942 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T16:48:25,942 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T16:48:25,942 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T16:48:25,942 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T16:48:25,943 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-15T16:48:25,943 DEBUG [StoreFileCloser-8b6a5bccf03f461ca7448a06c3e9e509-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/family/26c1092da2ba400fbabc94cb199754dc 2024-11-15T16:48:25,952 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnRegionMove1731689303497/8b6a5bccf03f461ca7448a06c3e9e509/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-15T16:48:25,953 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,953 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8b6a5bccf03f461ca7448a06c3e9e509: Waiting for close lock at 1731689305941Running coprocessor pre-close hooks at 1731689305941Disabling compacts and flushes for region at 1731689305941Disabling writes for close at 1731689305942 (+1 ms)Writing region close event to WAL at 1731689305944 (+2 ms)Running coprocessor post-close hooks at 1731689305952 (+8 ms)Closed at 1731689305952 2024-11-15T16:48:25,953 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509. 2024-11-15T16:48:25,955 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 53761e073339455dbb4391749fffd0f2 2024-11-15T16:48:25,974 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/info/53761e073339455dbb4391749fffd0f2 is 205, key is testBlockEvictionOnRegionMove1731689303497,,1731689303514.8b6a5bccf03f461ca7448a06c3e9e509./info:regioninfo/1731689305892/Put/seqid=0 2024-11-15T16:48:25,975 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/info/53761e073339455dbb4391749fffd0f2 2024-11-15T16:48:25,986 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d260bc60ff504862999c7b48da9a38dc 2024-11-15T16:48:26,001 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/ns/d260bc60ff504862999c7b48da9a38dc is 43, key is default/ns:d/1731689303220/Put/seqid=0 2024-11-15T16:48:26,001 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/ns/d260bc60ff504862999c7b48da9a38dc 2024-11-15T16:48:26,004 INFO [regionserver/a9bde5b3a1b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:26,004 INFO [regionserver/a9bde5b3a1b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:26,011 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 01e8665d2eb7485b96bed0cde1b60918 2024-11-15T16:48:26,025 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/table/01e8665d2eb7485b96bed0cde1b60918 is 78, key is testBlockEvictionOnRegionMove1731689303497/table:state/1731689303945/Put/seqid=0 2024-11-15T16:48:26,026 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/table/01e8665d2eb7485b96bed0cde1b60918 2024-11-15T16:48:26,029 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/info/53761e073339455dbb4391749fffd0f2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/53761e073339455dbb4391749fffd0f2 2024-11-15T16:48:26,030 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/53761e073339455dbb4391749fffd0f2, delay=919 ms 2024-11-15T16:48:26,032 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/53761e073339455dbb4391749fffd0f2, entries=15, sequenceid=15, filesize=7.6 K 2024-11-15T16:48:26,032 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/ns/d260bc60ff504862999c7b48da9a38dc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/d260bc60ff504862999c7b48da9a38dc 2024-11-15T16:48:26,034 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/d260bc60ff504862999c7b48da9a38dc, delay=906 ms 2024-11-15T16:48:26,035 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/d260bc60ff504862999c7b48da9a38dc, entries=2, sequenceid=15, filesize=5.0 K 2024-11-15T16:48:26,035 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/table/01e8665d2eb7485b96bed0cde1b60918 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/01e8665d2eb7485b96bed0cde1b60918 2024-11-15T16:48:26,037 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/01e8665d2eb7485b96bed0cde1b60918, delay=951 ms 2024-11-15T16:48:26,038 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/01e8665d2eb7485b96bed0cde1b60918, entries=2, sequenceid=15, filesize=5.3 K 2024-11-15T16:48:26,039 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 96ms, sequenceid=15, compaction requested=false 2024-11-15T16:48:26,039 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T16:48:26,040 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/53761e073339455dbb4391749fffd0f2 2024-11-15T16:48:26,041 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/d260bc60ff504862999c7b48da9a38dc 2024-11-15T16:48:26,042 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/01e8665d2eb7485b96bed0cde1b60918 2024-11-15T16:48:26,046 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-15T16:48:26,047 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T16:48:26,047 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T16:48:26,047 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731689305942Running coprocessor pre-close hooks at 1731689305942Disabling compacts and flushes for region at 1731689305942Disabling writes for close at 1731689305942Obtaining lock to block concurrent updates at 1731689305943 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731689305943Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731689305943Flushing stores of hbase:meta,,1.1588230740 at 1731689305944 (+1 ms)Flushing 1588230740/info: creating writer at 1731689305944Flushing 1588230740/info: appending metadata at 1731689305973 (+29 ms)Flushing 1588230740/info: closing flushed file at 1731689305973Flushing 1588230740/ns: creating writer at 1731689305978 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731689305999 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731689306000 (+1 ms)Flushing 1588230740/table: creating writer at 1731689306005 (+5 ms)Flushing 1588230740/table: appending metadata at 1731689306025 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731689306025Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21798667: reopening flushed file at 1731689306029 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f8e326c: reopening flushed file at 1731689306032 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d8b31ab: reopening flushed file at 1731689306035 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 96ms, sequenceid=15, compaction requested=false at 1731689306039 (+4 ms)Writing region close event to WAL at 1731689306042 (+3 ms)Running coprocessor post-close hooks at 1731689306047 (+5 ms)Closed at 1731689306047 2024-11-15T16:48:26,048 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T16:48:26,142 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(976): stopping server a9bde5b3a1b5,35489,1731689301089; all regions closed. 2024-11-15T16:48:26,142 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(976): stopping server a9bde5b3a1b5,37651,1731689301253; all regions closed. 2024-11-15T16:48:26,147 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs 2024-11-15T16:48:26,147 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs 2024-11-15T16:48:26,147 INFO [RS:0;a9bde5b3a1b5:35489 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C35489%2C1731689301089:(num 1731689302672) 2024-11-15T16:48:26,147 INFO [RS:1;a9bde5b3a1b5:37651 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C37651%2C1731689301253.meta:.meta(num 1731689303086) 2024-11-15T16:48:26,147 DEBUG [RS:0;a9bde5b3a1b5:35489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:26,147 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:26,147 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:48:26,147 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.ChoreService(370): Chore service for: regionserver/a9bde5b3a1b5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T16:48:26,148 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T16:48:26,148 INFO [regionserver/a9bde5b3a1b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:48:26,148 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T16:48:26,148 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T16:48:26,148 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:48:26,148 INFO [RS:0;a9bde5b3a1b5:35489 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35489 2024-11-15T16:48:26,149 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs 2024-11-15T16:48:26,149 INFO [RS:1;a9bde5b3a1b5:37651 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C37651%2C1731689301253:(num 1731689302671) 2024-11-15T16:48:26,149 DEBUG [RS:1;a9bde5b3a1b5:37651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:26,149 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:26,149 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:48:26,149 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.ChoreService(370): Chore service for: regionserver/a9bde5b3a1b5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T16:48:26,149 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:48:26,149 INFO [regionserver/a9bde5b3a1b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:48:26,150 INFO [RS:1;a9bde5b3a1b5:37651 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37651 2024-11-15T16:48:26,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:48:26,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a9bde5b3a1b5,35489,1731689301089 2024-11-15T16:48:26,153 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:48:26,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a9bde5b3a1b5,37651,1731689301253 2024-11-15T16:48:26,154 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:48:26,154 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a9bde5b3a1b5,37651,1731689301253] 2024-11-15T16:48:26,157 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a9bde5b3a1b5,37651,1731689301253 already deleted, retry=false 2024-11-15T16:48:26,157 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a9bde5b3a1b5,37651,1731689301253 expired; onlineServers=1 2024-11-15T16:48:26,157 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a9bde5b3a1b5,35489,1731689301089] 2024-11-15T16:48:26,159 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a9bde5b3a1b5,35489,1731689301089 already deleted, retry=false 2024-11-15T16:48:26,159 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a9bde5b3a1b5,35489,1731689301089 expired; onlineServers=0 2024-11-15T16:48:26,159 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a9bde5b3a1b5,33999,1731689300276' ***** 2024-11-15T16:48:26,159 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T16:48:26,160 INFO [M:0;a9bde5b3a1b5:33999 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:48:26,160 INFO [M:0;a9bde5b3a1b5:33999 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:48:26,160 DEBUG [M:0;a9bde5b3a1b5:33999 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T16:48:26,160 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T16:48:26,160 DEBUG [M:0;a9bde5b3a1b5:33999 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T16:48:26,160 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689302306 {}] cleaner.HFileCleaner(306): Exit Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689302306,5,FailOnTimeoutGroup] 2024-11-15T16:48:26,160 INFO [M:0;a9bde5b3a1b5:33999 {}] hbase.ChoreService(370): Chore service for: master/a9bde5b3a1b5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T16:48:26,160 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689302307 {}] cleaner.HFileCleaner(306): Exit Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689302307,5,FailOnTimeoutGroup] 2024-11-15T16:48:26,161 INFO [M:0;a9bde5b3a1b5:33999 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:48:26,161 DEBUG [M:0;a9bde5b3a1b5:33999 {}] master.HMaster(1795): Stopping service threads 2024-11-15T16:48:26,161 INFO [M:0;a9bde5b3a1b5:33999 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T16:48:26,161 INFO [M:0;a9bde5b3a1b5:33999 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T16:48:26,162 INFO [M:0;a9bde5b3a1b5:33999 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T16:48:26,162 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T16:48:26,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T16:48:26,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,163 DEBUG [M:0;a9bde5b3a1b5:33999 {}] zookeeper.ZKUtil(347): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T16:48:26,163 WARN [M:0;a9bde5b3a1b5:33999 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T16:48:26,164 INFO [M:0;a9bde5b3a1b5:33999 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/.lastflushedseqids 2024-11-15T16:48:26,174 INFO [M:0;a9bde5b3a1b5:33999 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T16:48:26,174 INFO [M:0;a9bde5b3a1b5:33999 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T16:48:26,174 DEBUG [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T16:48:26,174 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:26,174 DEBUG [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:26,174 DEBUG [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T16:48:26,174 DEBUG [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:26,175 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-15T16:48:26,186 DEBUG [M:0;a9bde5b3a1b5:33999 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7851e4ea62924dcf814bb1fb961b2493 2024-11-15T16:48:26,201 DEBUG [M:0;a9bde5b3a1b5:33999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7851e4ea62924dcf814bb1fb961b2493 is 82, key is hbase:meta,,1/info:regioninfo/1731689303158/Put/seqid=0 2024-11-15T16:48:26,202 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7851e4ea62924dcf814bb1fb961b2493 2024-11-15T16:48:26,213 DEBUG [M:0;a9bde5b3a1b5:33999 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 20fa27ae68ff47d28327cc5109da3257 2024-11-15T16:48:26,228 DEBUG [M:0;a9bde5b3a1b5:33999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/20fa27ae68ff47d28327cc5109da3257 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731689303950/Put/seqid=0 2024-11-15T16:48:26,230 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/20fa27ae68ff47d28327cc5109da3257 2024-11-15T16:48:26,238 DEBUG [M:0;a9bde5b3a1b5:33999 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a8c8a89eb78b46f78a7ed4f029536d86 2024-11-15T16:48:26,252 DEBUG [M:0;a9bde5b3a1b5:33999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a8c8a89eb78b46f78a7ed4f029536d86 is 69, key is a9bde5b3a1b5,35489,1731689301089/rs:state/1731689302379/Put/seqid=0 2024-11-15T16:48:26,253 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a8c8a89eb78b46f78a7ed4f029536d86 2024-11-15T16:48:26,255 DEBUG [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7851e4ea62924dcf814bb1fb961b2493 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7851e4ea62924dcf814bb1fb961b2493 2024-11-15T16:48:26,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:26,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35489-0x101acd535610001, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:26,256 INFO [RS:0;a9bde5b3a1b5:35489 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:48:26,257 INFO [RS:0;a9bde5b3a1b5:35489 {}] regionserver.HRegionServer(1031): Exiting; stopping=a9bde5b3a1b5,35489,1731689301089; zookeeper connection closed. 2024-11-15T16:48:26,257 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4046d876 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4046d876 2024-11-15T16:48:26,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:26,257 INFO [RS:1;a9bde5b3a1b5:37651 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:48:26,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37651-0x101acd535610002, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:26,258 INFO [RS:1;a9bde5b3a1b5:37651 {}] regionserver.HRegionServer(1031): Exiting; stopping=a9bde5b3a1b5,37651,1731689301253; zookeeper connection closed. 2024-11-15T16:48:26,258 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7851e4ea62924dcf814bb1fb961b2493, entries=8, sequenceid=84, filesize=5.5 K 2024-11-15T16:48:26,258 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6a93f920 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6a93f920 2024-11-15T16:48:26,258 DEBUG [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/20fa27ae68ff47d28327cc5109da3257 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/20fa27ae68ff47d28327cc5109da3257 2024-11-15T16:48:26,258 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-15T16:48:26,260 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/20fa27ae68ff47d28327cc5109da3257, entries=9, sequenceid=84, filesize=6.5 K 2024-11-15T16:48:26,260 DEBUG [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a8c8a89eb78b46f78a7ed4f029536d86 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a8c8a89eb78b46f78a7ed4f029536d86 2024-11-15T16:48:26,262 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a8c8a89eb78b46f78a7ed4f029536d86, entries=2, sequenceid=84, filesize=5.1 K 2024-11-15T16:48:26,263 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 88ms, sequenceid=84, compaction requested=false 2024-11-15T16:48:26,265 INFO [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:26,265 DEBUG [M:0;a9bde5b3a1b5:33999 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731689306174Disabling compacts and flushes for region at 1731689306174Disabling writes for close at 1731689306174Obtaining lock to block concurrent updates at 1731689306175 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731689306175Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731689306175Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731689306176 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731689306176Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731689306200 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731689306200Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731689306205 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731689306228 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731689306228Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731689306231 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731689306252 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731689306252Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@143c4e51: reopening flushed file at 1731689306255 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b59ee89: reopening flushed file at 1731689306258 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1089571b: reopening flushed file at 1731689306260 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 88ms, sequenceid=84, compaction requested=false at 1731689306263 (+3 ms)Writing region close event to WAL at 1731689306265 (+2 ms)Closed at 1731689306265 2024-11-15T16:48:26,267 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:48:26,267 INFO [M:0;a9bde5b3a1b5:33999 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T16:48:26,267 INFO [M:0;a9bde5b3a1b5:33999 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33999 2024-11-15T16:48:26,267 INFO [M:0;a9bde5b3a1b5:33999 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:48:26,370 INFO [M:0;a9bde5b3a1b5:33999 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:48:26,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:26,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33999-0x101acd535610000, quorum=127.0.0.1:49978, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:26,376 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T16:48:26,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-15T16:48:26,398 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:26,407 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: master/a9bde5b3a1b5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/a9bde5b3a1b5:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731689305877 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731689304340 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731689306034 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a9bde5b3a1b5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731689306030 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/a9bde5b3a1b5:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1235 (was 1387), ProcessCount=11 (was 11), AvailableMemoryMB=6119 (was 6511) 2024-11-15T16:48:26,413 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=1235, ProcessCount=11, AvailableMemoryMB=6120 2024-11-15T16:48:26,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/hadoop-log-dir so I do NOT create it in target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae 2024-11-15T16:48:26,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/hadoop-tmp-dir so I do NOT create it in target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae 2024-11-15T16:48:26,414 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:26,417 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/cluster_a1c0743a-c5a5-cb31-41dc-8494a312854b, deleteOnExit=true 2024-11-15T16:48:26,420 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/cluster_a1c0743a-c5a5-cb31-41dc-8494a312854b/zookeeper_0, clientPort=60559, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/cluster_a1c0743a-c5a5-cb31-41dc-8494a312854b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/cluster_a1c0743a-c5a5-cb31-41dc-8494a312854b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T16:48:26,421 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60559 2024-11-15T16:48:26,433 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 with version=8 2024-11-15T16:48:26,433 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/hbase-staging 2024-11-15T16:48:26,435 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:48:26,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:26,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:26,436 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:48:26,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:26,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:48:26,436 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T16:48:26,436 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:48:26,437 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39189 2024-11-15T16:48:26,438 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39189 connecting to ZooKeeper ensemble=127.0.0.1:60559 2024-11-15T16:48:26,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:391890x0, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:48:26,445 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39189-0x101acd54e9b0000 connected 2024-11-15T16:48:26,467 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:26,468 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:26,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:26,471 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913, hbase.cluster.distributed=false 2024-11-15T16:48:26,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:48:26,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39189 2024-11-15T16:48:26,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39189 2024-11-15T16:48:26,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39189 2024-11-15T16:48:26,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39189 2024-11-15T16:48:26,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39189 2024-11-15T16:48:26,493 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:48:26,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:26,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:26,494 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:48:26,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:26,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:48:26,494 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T16:48:26,494 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:48:26,495 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41911 2024-11-15T16:48:26,496 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41911 connecting to ZooKeeper ensemble=127.0.0.1:60559 2024-11-15T16:48:26,496 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:26,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:26,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419110x0, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:48:26,500 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:419110x0, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:26,501 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T16:48:26,501 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41911-0x101acd54e9b0001 connected 2024-11-15T16:48:26,501 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.cache 2024-11-15T16:48:26,501 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T16:48:26,502 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T16:48:26,502 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T16:48:26,502 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T16:48:26,502 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-15T16:48:26,502 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T16:48:26,502 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.persistence is complete. 2024-11-15T16:48:26,503 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T16:48:26,503 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T16:48:26,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T16:48:26,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:48:26,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41911 2024-11-15T16:48:26,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41911 2024-11-15T16:48:26,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41911 2024-11-15T16:48:26,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41911 2024-11-15T16:48:26,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41911 2024-11-15T16:48:26,530 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:48:26,531 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:26,531 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:26,531 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:48:26,531 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:26,531 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:48:26,531 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T16:48:26,531 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:48:26,532 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41733 2024-11-15T16:48:26,534 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41733 connecting to ZooKeeper ensemble=127.0.0.1:60559 2024-11-15T16:48:26,534 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:26,535 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:26,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417330x0, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:48:26,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41733-0x101acd54e9b0002 connected 2024-11-15T16:48:26,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:26,540 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T16:48:26,544 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.cache 2024-11-15T16:48:26,544 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T16:48:26,545 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T16:48:26,545 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T16:48:26,545 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T16:48:26,545 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-15T16:48:26,545 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T16:48:26,545 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.persistence is complete. 2024-11-15T16:48:26,548 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T16:48:26,549 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T16:48:26,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T16:48:26,551 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:48:26,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41733 2024-11-15T16:48:26,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41733 2024-11-15T16:48:26,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41733 2024-11-15T16:48:26,556 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41733 2024-11-15T16:48:26,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41733 2024-11-15T16:48:26,580 DEBUG [M:0;a9bde5b3a1b5:39189 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a9bde5b3a1b5:39189 2024-11-15T16:48:26,581 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:26,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:26,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:26,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:26,586 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:26,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T16:48:26,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T16:48:26,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,590 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T16:48:26,590 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a9bde5b3a1b5,39189,1731689306435 from backup master directory 2024-11-15T16:48:26,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:26,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:26,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:26,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:48:26,593 WARN [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:48:26,593 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:26,595 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/hbase.id] with ID: 9ae5ac53-b7f5-40a5-b07b-9a331e5f2620 2024-11-15T16:48:26,595 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/.tmp/hbase.id 2024-11-15T16:48:26,603 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/hbase.id] 2024-11-15T16:48:26,646 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:26,646 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T16:48:26,648 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-15T16:48:26,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,662 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:48:26,663 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T16:48:26,664 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:26,685 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store 2024-11-15T16:48:26,691 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:26,692 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T16:48:26,692 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:26,692 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:26,692 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T16:48:26,692 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:26,692 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:26,692 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731689306692Disabling compacts and flushes for region at 1731689306692Disabling writes for close at 1731689306692Writing region close event to WAL at 1731689306692Closed at 1731689306692 2024-11-15T16:48:26,692 WARN [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/.initializing 2024-11-15T16:48:26,692 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/WALs/a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:26,698 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:48:26,705 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C39189%2C1731689306435, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/WALs/a9bde5b3a1b5,39189,1731689306435, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/oldWALs, maxLogs=10 2024-11-15T16:48:26,724 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/WALs/a9bde5b3a1b5,39189,1731689306435/a9bde5b3a1b5%2C39189%2C1731689306435.1731689306705 2024-11-15T16:48:26,725 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:26,725 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:26,725 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:26,725 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,726 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,732 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T16:48:26,732 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:26,732 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:26,733 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T16:48:26,737 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:26,737 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:26,737 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T16:48:26,741 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:26,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:26,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T16:48:26,745 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:26,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:26,745 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,746 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,746 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,746 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,746 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,746 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:48:26,747 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:48:26,754 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:48:26,754 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63439879, jitterRate=-0.05467213690280914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:48:26,756 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731689306726Initializing all the Stores at 1731689306726Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689306726Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689306726Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689306726Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689306726Cleaning up temporary data from old regions at 1731689306746 (+20 ms)Region opened successfully at 1731689306755 (+9 ms) 2024-11-15T16:48:26,756 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T16:48:26,761 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21079121, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:48:26,763 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T16:48:26,763 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T16:48:26,763 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T16:48:26,763 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T16:48:26,764 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T16:48:26,764 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-15T16:48:26,765 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T16:48:26,767 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T16:48:26,769 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T16:48:26,771 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T16:48:26,771 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T16:48:26,772 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T16:48:26,774 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T16:48:26,774 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T16:48:26,776 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T16:48:26,777 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T16:48:26,779 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T16:48:26,780 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T16:48:26,783 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T16:48:26,785 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T16:48:26,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:26,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:26,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:26,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,788 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a9bde5b3a1b5,39189,1731689306435, sessionid=0x101acd54e9b0000, setting cluster-up flag (Was=false) 2024-11-15T16:48:26,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,797 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T16:48:26,798 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:26,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:26,808 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T16:48:26,809 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:26,810 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T16:48:26,813 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T16:48:26,813 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T16:48:26,813 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T16:48:26,814 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a9bde5b3a1b5,39189,1731689306435 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T16:48:26,816 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:48:26,816 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:48:26,816 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:48:26,816 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:48:26,816 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a9bde5b3a1b5:0, corePoolSize=10, maxPoolSize=10 2024-11-15T16:48:26,816 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,816 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:48:26,817 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,818 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731689336818 2024-11-15T16:48:26,818 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T16:48:26,818 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T16:48:26,818 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T16:48:26,818 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T16:48:26,818 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T16:48:26,818 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:48:26,818 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T16:48:26,819 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T16:48:26,819 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:26,819 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T16:48:26,820 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,820 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T16:48:26,820 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T16:48:26,821 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T16:48:26,824 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T16:48:26,825 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T16:48:26,825 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689306825,5,FailOnTimeoutGroup] 2024-11-15T16:48:26,825 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689306825,5,FailOnTimeoutGroup] 2024-11-15T16:48:26,825 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,825 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T16:48:26,825 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,825 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,839 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T16:48:26,839 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 2024-11-15T16:48:26,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:26,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-15T16:48:26,852 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:48:26,852 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:26,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:26,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-15T16:48:26,856 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:48:26,856 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:26,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:26,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-15T16:48:26,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:48:26,860 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:26,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:26,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-15T16:48:26,862 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(746): ClusterId : 9ae5ac53-b7f5-40a5-b07b-9a331e5f2620 2024-11-15T16:48:26,862 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T16:48:26,862 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(746): ClusterId : 9ae5ac53-b7f5-40a5-b07b-9a331e5f2620 2024-11-15T16:48:26,862 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T16:48:26,864 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:48:26,864 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:26,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:26,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:48:26,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:26,866 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:26,866 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:48:26,866 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:48:26,866 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:48:26,866 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T16:48:26,866 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T16:48:26,867 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:48:26,869 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T16:48:26,870 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce87a52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:48:26,875 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:48:26,875 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59785613, jitterRate=-0.10912494361400604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:48:26,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731689306847Initializing all the Stores at 1731689306847Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689306847Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689306847Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689306847Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689306847Cleaning up temporary data from old regions at 1731689306866 (+19 ms)Region opened successfully at 1731689306876 (+10 ms) 2024-11-15T16:48:26,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T16:48:26,876 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T16:48:26,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T16:48:26,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T16:48:26,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T16:48:26,876 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T16:48:26,877 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T16:48:26,877 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T16:48:26,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731689306876Disabling compacts and flushes for region at 1731689306876Disabling writes for close at 1731689306876Writing region close event to WAL at 1731689306877 (+1 ms)Closed at 1731689306877 2024-11-15T16:48:26,878 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:48:26,879 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T16:48:26,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T16:48:26,880 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T16:48:26,880 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:48:26,880 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251d4ee5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:48:26,881 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T16:48:26,887 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a9bde5b3a1b5:41733 2024-11-15T16:48:26,887 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T16:48:26,887 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T16:48:26,887 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T16:48:26,888 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,39189,1731689306435 with port=41733, startcode=1731689306530 2024-11-15T16:48:26,888 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T16:48:26,894 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38455, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T16:48:26,895 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:26,895 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(517): Registering regionserver=a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:26,897 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 2024-11-15T16:48:26,897 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T16:48:26,897 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T16:48:26,897 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a9bde5b3a1b5:41911 2024-11-15T16:48:26,898 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T16:48:26,898 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T16:48:26,898 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T16:48:26,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:48:26,899 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,39189,1731689306435 with port=41911, startcode=1731689306493 2024-11-15T16:48:26,900 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T16:48:26,901 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] zookeeper.ZKUtil(111): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:26,901 WARN [RS:1;a9bde5b3a1b5:41733 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:48:26,901 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a9bde5b3a1b5,41733,1731689306530] 2024-11-15T16:48:26,901 INFO [RS:1;a9bde5b3a1b5:41733 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:26,901 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:26,906 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45375, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T16:48:26,907 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:26,907 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(517): Registering regionserver=a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:26,907 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T16:48:26,909 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 2024-11-15T16:48:26,909 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T16:48:26,909 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T16:48:26,910 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T16:48:26,912 INFO [RS:1;a9bde5b3a1b5:41733 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:48:26,912 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,912 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T16:48:26,913 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T16:48:26,913 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,913 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,914 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,915 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,915 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:26,915 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:26,915 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,915 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,915 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,915 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,915 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,916 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41733,1731689306530-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:48:26,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:48:26,919 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] zookeeper.ZKUtil(111): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:26,919 WARN [RS:0;a9bde5b3a1b5:41911 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:48:26,919 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a9bde5b3a1b5,41911,1731689306493] 2024-11-15T16:48:26,919 INFO [RS:0;a9bde5b3a1b5:41911 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:26,919 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:26,926 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T16:48:26,928 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T16:48:26,930 INFO [RS:0;a9bde5b3a1b5:41911 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:48:26,930 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,931 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T16:48:26,931 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T16:48:26,931 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,931 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,932 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,932 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,932 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,932 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,932 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:48:26,932 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,932 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T16:48:26,932 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,932 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41733,1731689306530-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,932 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,932 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,933 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,933 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,933 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.Replication(171): a9bde5b3a1b5,41733,1731689306530 started 2024-11-15T16:48:26,933 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:26,933 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:26,933 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:26,936 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,936 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,936 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,936 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,936 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,936 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41911,1731689306493-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:48:26,954 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,954 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(1482): Serving as a9bde5b3a1b5,41733,1731689306530, RpcServer on a9bde5b3a1b5/172.17.0.2:41733, sessionid=0x101acd54e9b0002 2024-11-15T16:48:26,954 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T16:48:26,954 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:26,954 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,41733,1731689306530' 2024-11-15T16:48:26,954 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T16:48:26,955 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T16:48:26,956 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T16:48:26,956 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T16:48:26,956 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:26,956 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,41733,1731689306530' 2024-11-15T16:48:26,956 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T16:48:26,957 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T16:48:26,957 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T16:48:26,957 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41911,1731689306493-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,957 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T16:48:26,957 INFO [RS:1;a9bde5b3a1b5:41733 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T16:48:26,957 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,957 INFO [RS:1;a9bde5b3a1b5:41733 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T16:48:26,958 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.Replication(171): a9bde5b3a1b5,41911,1731689306493 started 2024-11-15T16:48:26,972 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:26,973 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(1482): Serving as a9bde5b3a1b5,41911,1731689306493, RpcServer on a9bde5b3a1b5/172.17.0.2:41911, sessionid=0x101acd54e9b0001 2024-11-15T16:48:26,973 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T16:48:26,973 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:26,973 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,41911,1731689306493' 2024-11-15T16:48:26,973 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T16:48:26,974 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T16:48:26,974 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T16:48:26,974 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T16:48:26,974 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:26,974 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,41911,1731689306493' 2024-11-15T16:48:26,974 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T16:48:26,975 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T16:48:26,975 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T16:48:26,975 INFO [RS:0;a9bde5b3a1b5:41911 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T16:48:26,975 INFO [RS:0;a9bde5b3a1b5:41911 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T16:48:27,032 WARN [a9bde5b3a1b5:39189 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-15T16:48:27,058 INFO [RS:1;a9bde5b3a1b5:41733 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:48:27,059 INFO [RS:1;a9bde5b3a1b5:41733 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a9bde5b3a1b5%2C41733%2C1731689306530, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41733,1731689306530, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs, maxLogs=55 2024-11-15T16:48:27,076 INFO [RS:0;a9bde5b3a1b5:41911 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:48:27,076 INFO [RS:0;a9bde5b3a1b5:41911 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a9bde5b3a1b5%2C41911%2C1731689306493, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs, maxLogs=55 2024-11-15T16:48:27,079 INFO [RS:1;a9bde5b3a1b5:41733 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41733,1731689306530/a9bde5b3a1b5%2C41733%2C1731689306530.1731689307061 2024-11-15T16:48:27,080 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:27,097 INFO [RS:0;a9bde5b3a1b5:41911 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493/a9bde5b3a1b5%2C41911%2C1731689306493.1731689307077 2024-11-15T16:48:27,097 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:27,282 DEBUG [a9bde5b3a1b5:39189 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-15T16:48:27,283 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(204): Hosts are {a9bde5b3a1b5=0} racks are {/default-rack=0} 2024-11-15T16:48:27,283 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T16:48:27,283 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T16:48:27,283 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T16:48:27,283 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T16:48:27,283 INFO [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T16:48:27,284 INFO [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T16:48:27,284 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T16:48:27,284 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:27,286 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,41911,1731689306493, state=OPENING 2024-11-15T16:48:27,288 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T16:48:27,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:27,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:27,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:27,290 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:48:27,290 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:27,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,41911,1731689306493}] 2024-11-15T16:48:27,290 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:27,292 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:27,444 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:48:27,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42071, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:48:27,449 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T16:48:27,449 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:27,449 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T16:48:27,450 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a9bde5b3a1b5%2C41911%2C1731689306493.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs, maxLogs=55 2024-11-15T16:48:27,471 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493/a9bde5b3a1b5%2C41911%2C1731689306493.meta.1731689307451.meta 2024-11-15T16:48:27,472 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:27,472 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:27,473 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T16:48:27,473 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T16:48:27,473 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T16:48:27,474 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T16:48:27,474 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:27,474 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T16:48:27,474 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T16:48:27,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-15T16:48:27,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:48:27,476 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:27,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:27,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-15T16:48:27,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:48:27,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:27,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:27,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-15T16:48:27,478 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:48:27,478 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:27,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:27,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-15T16:48:27,479 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:48:27,479 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:27,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:27,479 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:48:27,480 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:27,480 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:27,481 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:48:27,481 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:48:27,481 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:48:27,482 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:48:27,483 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69390855, jitterRate=0.03400431573390961}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:48:27,483 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T16:48:27,484 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731689307474Writing region info on filesystem at 1731689307474Initializing all the Stores at 1731689307475 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689307475Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689307475Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689307475Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689307475Cleaning up temporary data from old regions at 1731689307481 (+6 ms)Running coprocessor post-open hooks at 1731689307483 (+2 ms)Region opened successfully at 1731689307484 (+1 ms) 2024-11-15T16:48:27,485 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731689307443 2024-11-15T16:48:27,488 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T16:48:27,488 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T16:48:27,488 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:27,489 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,41911,1731689306493, state=OPEN 2024-11-15T16:48:27,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:27,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:27,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:27,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:27,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:27,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:27,493 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:27,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T16:48:27,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,41911,1731689306493 in 203 msec 2024-11-15T16:48:27,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T16:48:27,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 616 msec 2024-11-15T16:48:27,498 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:48:27,498 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T16:48:27,502 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:48:27,502 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,41911,1731689306493, seqNum=-1] 2024-11-15T16:48:27,502 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:48:27,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:48:27,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 698 msec 2024-11-15T16:48:27,511 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731689307511, completionTime=-1 2024-11-15T16:48:27,511 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-15T16:48:27,511 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T16:48:27,514 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T16:48:27,514 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731689367514 2024-11-15T16:48:27,514 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731689427514 2024-11-15T16:48:27,514 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-15T16:48:27,514 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-15T16:48:27,515 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,39189,1731689306435-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:27,515 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,39189,1731689306435-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:27,515 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,39189,1731689306435-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:27,515 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a9bde5b3a1b5:39189, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:27,515 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:27,516 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:27,518 DEBUG [master/a9bde5b3a1b5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T16:48:27,521 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.927sec 2024-11-15T16:48:27,521 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T16:48:27,521 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T16:48:27,521 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T16:48:27,521 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T16:48:27,521 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T16:48:27,521 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,39189,1731689306435-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:48:27,521 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,39189,1731689306435-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T16:48:27,522 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T16:48:27,522 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T16:48:27,522 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,39189,1731689306435-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:27,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:48:27,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,39189,-1 for getting cluster id 2024-11-15T16:48:27,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:48:27,565 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9ae5ac53-b7f5-40a5-b07b-9a331e5f2620' 2024-11-15T16:48:27,565 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:48:27,566 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9ae5ac53-b7f5-40a5-b07b-9a331e5f2620" 2024-11-15T16:48:27,566 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d6690f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:48:27,566 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,39189,-1] 2024-11-15T16:48:27,566 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:48:27,567 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:27,569 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:48:27,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:48:27,571 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:48:27,572 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,41911,1731689306493, seqNum=-1] 2024-11-15T16:48:27,573 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:48:27,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:48:27,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:27,578 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T16:48:27,579 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:27,579 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fee78e 2024-11-15T16:48:27,580 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T16:48:27,582 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T16:48:27,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731689307577', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:48:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731689307577 2024-11-15T16:48:27,585 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731689307577 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T16:48:27,585 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:27,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731689307577" procId is: 4 2024-11-15T16:48:27,587 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731689307577 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T16:48:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:48:27,599 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731689307577-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cd47fcba664f01421cc5a2b65b6cbbce, NAME => 'testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731689307577', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 2024-11-15T16:48:27,607 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731689307577-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:27,607 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731689307577-pool-0 {}] regionserver.HRegion(1722): Closing cd47fcba664f01421cc5a2b65b6cbbce, disabling compactions & flushes 2024-11-15T16:48:27,607 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731689307577-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:27,607 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731689307577-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:27,608 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731689307577-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. after waiting 0 ms 2024-11-15T16:48:27,608 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731689307577-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:27,608 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731689307577-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:27,608 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731689307577-pool-0 {}] regionserver.HRegion(1676): Region close journal for cd47fcba664f01421cc5a2b65b6cbbce: Waiting for close lock at 1731689307607Disabling compacts and flushes for region at 1731689307607Disabling writes for close at 1731689307608 (+1 ms)Writing region close event to WAL at 1731689307608Closed at 1731689307608 2024-11-15T16:48:27,609 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731689307577 execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T16:48:27,610 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731689307609"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731689307609"}]},"ts":"1731689307609"} 2024-11-15T16:48:27,612 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T16:48:27,613 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731689307577 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T16:48:27,614 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731689307577","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689307613"}]},"ts":"1731689307613"} 2024-11-15T16:48:27,616 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731689307577, state=ENABLING in hbase:meta 2024-11-15T16:48:27,616 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a9bde5b3a1b5=0} racks are {/default-rack=0} 2024-11-15T16:48:27,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T16:48:27,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T16:48:27,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T16:48:27,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T16:48:27,617 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T16:48:27,617 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T16:48:27,617 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T16:48:27,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN}] 2024-11-15T16:48:27,618 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN 2024-11-15T16:48:27,619 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN; state=OFFLINE, location=a9bde5b3a1b5,41911,1731689306493; forceNewPlan=false, retain=false 2024-11-15T16:48:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:48:27,770 INFO [a9bde5b3a1b5:39189 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T16:48:27,770 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cd47fcba664f01421cc5a2b65b6cbbce, regionState=OPENING, regionLocation=a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:27,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN because future has completed 2024-11-15T16:48:27,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd47fcba664f01421cc5a2b65b6cbbce, server=a9bde5b3a1b5,41911,1731689306493}] 2024-11-15T16:48:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:48:27,929 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:27,930 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cd47fcba664f01421cc5a2b65b6cbbce, NAME => 'testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:27,930 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731689307577 cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,930 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:27,930 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,931 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,931 INFO [StoreOpener-cd47fcba664f01421cc5a2b65b6cbbce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,936 INFO [StoreOpener-cd47fcba664f01421cc5a2b65b6cbbce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd47fcba664f01421cc5a2b65b6cbbce columnFamilyName family 2024-11-15T16:48:27,936 DEBUG [StoreOpener-cd47fcba664f01421cc5a2b65b6cbbce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:27,936 INFO [StoreOpener-cd47fcba664f01421cc5a2b65b6cbbce-1 {}] regionserver.HStore(327): Store=cd47fcba664f01421cc5a2b65b6cbbce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:27,937 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,937 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,938 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,938 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,938 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,939 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,945 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:48:27,946 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cd47fcba664f01421cc5a2b65b6cbbce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62976122, jitterRate=-0.06158265471458435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:48:27,946 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:27,947 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cd47fcba664f01421cc5a2b65b6cbbce: Running coprocessor pre-open hook at 1731689307931Writing region info on filesystem at 1731689307931Initializing all the Stores at 1731689307931Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689307931Cleaning up temporary data from old regions at 1731689307938 (+7 ms)Running coprocessor post-open hooks at 1731689307946 (+8 ms)Region opened successfully at 1731689307947 (+1 ms) 2024-11-15T16:48:27,948 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce., pid=6, masterSystemTime=1731689307927 2024-11-15T16:48:27,950 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:27,951 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:27,951 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cd47fcba664f01421cc5a2b65b6cbbce, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:27,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd47fcba664f01421cc5a2b65b6cbbce, server=a9bde5b3a1b5,41911,1731689306493 because future has completed 2024-11-15T16:48:27,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T16:48:27,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cd47fcba664f01421cc5a2b65b6cbbce, server=a9bde5b3a1b5,41911,1731689306493 in 181 msec 2024-11-15T16:48:27,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T16:48:27,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN in 340 msec 2024-11-15T16:48:27,960 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731689307577 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T16:48:27,960 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731689307577","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689307960"}]},"ts":"1731689307960"} 2024-11-15T16:48:27,963 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731689307577, state=ENABLED in hbase:meta 2024-11-15T16:48:27,964 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731689307577 execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T16:48:27,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731689307577 in 381 msec 2024-11-15T16:48:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:48:28,213 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731689307577 completed 2024-11-15T16:48:28,213 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731689307577 get assigned. Timeout = 60000ms 2024-11-15T16:48:28,213 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T16:48:28,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731689307577 assigned to meta. Checking AM states. 2024-11-15T16:48:28,218 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T16:48:28,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731689307577 assigned. 2024-11-15T16:48:28,222 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731689307577', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce., hostname=a9bde5b3a1b5,41911,1731689306493, seqNum=2] 2024-11-15T16:48:28,228 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cd47fcba664f01421cc5a2b65b6cbbce 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-15T16:48:28,239 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1e182b5c0589431ebc3bfcf641de6770 2024-11-15T16:48:28,257 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/.tmp/family/1e182b5c0589431ebc3bfcf641de6770 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-15T16:48:28,258 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/.tmp/family/1e182b5c0589431ebc3bfcf641de6770 2024-11-15T16:48:28,262 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/.tmp/family/1e182b5c0589431ebc3bfcf641de6770 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770 2024-11-15T16:48:28,264 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770, delay=920 ms 2024-11-15T16:48:28,265 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770, entries=2, sequenceid=6, filesize=4.9 K 2024-11-15T16:48:28,266 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for cd47fcba664f01421cc5a2b65b6cbbce in 38ms, sequenceid=6, compaction requested=false 2024-11-15T16:48:28,266 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731689307577' 2024-11-15T16:48:28,267 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cd47fcba664f01421cc5a2b65b6cbbce: 2024-11-15T16:48:28,624 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T16:48:29,185 DEBUG [hfile-prefetch-1731689305877 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1e182b5c0589431ebc3bfcf641de6770_0. 2024-11-15T16:48:29,186 DEBUG [hfile-prefetch-1731689305877 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1e182b5c0589431ebc3bfcf641de6770_117. 2024-11-15T16:48:29,186 DEBUG [hfile-prefetch-1731689305877 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770, with total blocks 2, and data blocks 1 2024-11-15T16:48:29,186 DEBUG [hfile-prefetch-1731689305877 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-15T16:48:29,186 DEBUG [hfile-prefetch-1731689305877 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-15T16:48:29,186 DEBUG [hfile-prefetch-1731689305877 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770. Will sleep 100ms and try the verification again. 2024-11-15T16:48:29,204 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-15T16:48:29,204 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-15T16:48:29,204 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:29,204 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:29,208 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-15T16:48:29,208 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731689309204 2024-11-15T16:48:29,208 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-15T16:48:29,268 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;a9bde5b3a1b5:41911,5,FailOnTimeoutGroup] 2024-11-15T16:48:29,268 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a9bde5b3a1b5,41911,1731689306493' ***** 2024-11-15T16:48:29,268 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-15T16:48:29,268 INFO [RS:0;a9bde5b3a1b5:41911 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T16:48:29,268 INFO [RS:0;a9bde5b3a1b5:41911 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.persistence 2024-11-15T16:48:29,268 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T16:48:29,268 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T16:48:29,268 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T16:48:29,268 INFO [RS:0;a9bde5b3a1b5:41911 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T16:48:29,269 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T16:48:29,279 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] bucket.BucketCache(1487): Thread RS:0;a9bde5b3a1b5:41911 started persisting bucket cache to file 2024-11-15T16:48:29,279 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:29,279 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:29,282 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-15T16:48:29,282 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731689309279 2024-11-15T16:48:29,282 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] bucket.BucketCache(1509): Thread RS:0;a9bde5b3a1b5:41911 finished persisting bucket cache to file, renaming 2024-11-15T16:48:29,283 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T16:48:29,283 INFO [RS:0;a9bde5b3a1b5:41911 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T16:48:29,283 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T16:48:29,283 INFO [RS:0;a9bde5b3a1b5:41911 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T16:48:29,283 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(3091): Received CLOSE for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:29,283 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(959): stopping server a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:29,283 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:48:29,283 INFO [RS:0;a9bde5b3a1b5:41911 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a9bde5b3a1b5:41911. 2024-11-15T16:48:29,283 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T16:48:29,284 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:48:29,284 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cd47fcba664f01421cc5a2b65b6cbbce, disabling compactions & flushes 2024-11-15T16:48:29,284 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:29,284 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:29,284 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:29,284 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T16:48:29,284 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. after waiting 0 ms 2024-11-15T16:48:29,284 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T16:48:29,284 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:29,284 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T16:48:29,284 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T16:48:29,284 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-15T16:48:29,284 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, cd47fcba664f01421cc5a2b65b6cbbce=testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce.} 2024-11-15T16:48:29,285 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:29,285 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T16:48:29,285 DEBUG [StoreFileCloser-cd47fcba664f01421cc5a2b65b6cbbce-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770 2024-11-15T16:48:29,285 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T16:48:29,285 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T16:48:29,285 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T16:48:29,285 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T16:48:29,285 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T16:48:29,285 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-15T16:48:29,287 WARN [hfile-prefetch-1731689305877 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-15T16:48:29,288 DEBUG [hfile-prefetch-1731689305877 {}] hfile.PrefetchExecutor(121): Prefetch completed for 1e182b5c0589431ebc3bfcf641de6770 2024-11-15T16:48:29,295 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-15T16:48:29,295 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:29,295 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cd47fcba664f01421cc5a2b65b6cbbce: Waiting for close lock at 1731689309283Running coprocessor pre-close hooks at 1731689309284 (+1 ms)Disabling compacts and flushes for region at 1731689309284Disabling writes for close at 1731689309284Writing region close event to WAL at 1731689309285 (+1 ms)Running coprocessor post-close hooks at 1731689309295 (+10 ms)Closed at 1731689309295 2024-11-15T16:48:29,295 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:29,299 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 07789c30693b4527bf3c85f6be86ad32 2024-11-15T16:48:29,313 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/info/07789c30693b4527bf3c85f6be86ad32 is 209, key is testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce./info:regioninfo/1731689307951/Put/seqid=0 2024-11-15T16:48:29,314 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/info/07789c30693b4527bf3c85f6be86ad32 2024-11-15T16:48:29,323 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 76d633d74e0a45e29bdc081a0d8f9dcc 2024-11-15T16:48:29,338 INFO [regionserver/a9bde5b3a1b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:29,342 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/ns/76d633d74e0a45e29bdc081a0d8f9dcc is 43, key is default/ns:d/1731689307505/Put/seqid=0 2024-11-15T16:48:29,342 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/ns/76d633d74e0a45e29bdc081a0d8f9dcc 2024-11-15T16:48:29,353 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8dcfe96637dd42baa22a8837c786e16f 2024-11-15T16:48:29,367 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/table/8dcfe96637dd42baa22a8837c786e16f is 80, key is testBlockEvictionOnGracefulStop1731689307577/table:state/1731689307960/Put/seqid=0 2024-11-15T16:48:29,367 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/table/8dcfe96637dd42baa22a8837c786e16f 2024-11-15T16:48:29,370 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/info/07789c30693b4527bf3c85f6be86ad32 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/07789c30693b4527bf3c85f6be86ad32 2024-11-15T16:48:29,372 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/07789c30693b4527bf3c85f6be86ad32, delay=901 ms 2024-11-15T16:48:29,373 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/07789c30693b4527bf3c85f6be86ad32, entries=10, sequenceid=11, filesize=7.0 K 2024-11-15T16:48:29,374 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/ns/76d633d74e0a45e29bdc081a0d8f9dcc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/76d633d74e0a45e29bdc081a0d8f9dcc 2024-11-15T16:48:29,375 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/76d633d74e0a45e29bdc081a0d8f9dcc, delay=986 ms 2024-11-15T16:48:29,376 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/76d633d74e0a45e29bdc081a0d8f9dcc, entries=2, sequenceid=11, filesize=5.0 K 2024-11-15T16:48:29,377 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/table/8dcfe96637dd42baa22a8837c786e16f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/8dcfe96637dd42baa22a8837c786e16f 2024-11-15T16:48:29,378 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/8dcfe96637dd42baa22a8837c786e16f, delay=946 ms 2024-11-15T16:48:29,378 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/8dcfe96637dd42baa22a8837c786e16f, entries=2, sequenceid=11, filesize=5.3 K 2024-11-15T16:48:29,379 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=11, compaction requested=false 2024-11-15T16:48:29,381 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/07789c30693b4527bf3c85f6be86ad32 2024-11-15T16:48:29,381 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/76d633d74e0a45e29bdc081a0d8f9dcc 2024-11-15T16:48:29,383 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/8dcfe96637dd42baa22a8837c786e16f 2024-11-15T16:48:29,389 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-15T16:48:29,390 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T16:48:29,390 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T16:48:29,390 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731689309285Running coprocessor pre-close hooks at 1731689309285Disabling compacts and flushes for region at 1731689309285Disabling writes for close at 1731689309285Obtaining lock to block concurrent updates at 1731689309285Preparing flush snapshotting stores in 1588230740 at 1731689309285Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731689309286 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731689309286Flushing 1588230740/info: creating writer at 1731689309287 (+1 ms)Flushing 1588230740/info: appending metadata at 1731689309313 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731689309313Flushing 1588230740/ns: creating writer at 1731689309316 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731689309341 (+25 ms)Flushing 1588230740/ns: closing flushed file at 1731689309341Flushing 1588230740/table: creating writer at 1731689309345 (+4 ms)Flushing 1588230740/table: appending metadata at 1731689309366 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731689309366Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24104a9e: reopening flushed file at 1731689309370 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51e99072: reopening flushed file at 1731689309374 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d7e99eb: reopening flushed file at 1731689309376 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=11, compaction requested=false at 1731689309379 (+3 ms)Writing region close event to WAL at 1731689309384 (+5 ms)Running coprocessor post-close hooks at 1731689309390 (+6 ms)Closed at 1731689309390 2024-11-15T16:48:29,390 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T16:48:29,485 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(976): stopping server a9bde5b3a1b5,41911,1731689306493; all regions closed. 2024-11-15T16:48:29,487 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs 2024-11-15T16:48:29,487 INFO [RS:0;a9bde5b3a1b5:41911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C41911%2C1731689306493.meta:.meta(num 1731689307451) 2024-11-15T16:48:29,489 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs 2024-11-15T16:48:29,489 INFO [RS:0;a9bde5b3a1b5:41911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C41911%2C1731689306493:(num 1731689307077) 2024-11-15T16:48:29,489 DEBUG [RS:0;a9bde5b3a1b5:41911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:29,489 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:29,490 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:48:29,490 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.ChoreService(370): Chore service for: regionserver/a9bde5b3a1b5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T16:48:29,490 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:48:29,490 INFO [regionserver/a9bde5b3a1b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:48:29,490 INFO [RS:0;a9bde5b3a1b5:41911 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41911 2024-11-15T16:48:29,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a9bde5b3a1b5,41911,1731689306493 2024-11-15T16:48:29,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:48:29,496 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:48:29,497 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a9bde5b3a1b5,41911,1731689306493] 2024-11-15T16:48:29,499 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a9bde5b3a1b5,41911,1731689306493 already deleted, retry=false 2024-11-15T16:48:29,499 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a9bde5b3a1b5,41911,1731689306493 on a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:29,504 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a9bde5b3a1b5,41911,1731689306493, splitWal=true, meta=true 2024-11-15T16:48:29,506 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for a9bde5b3a1b5,41911,1731689306493 (carryingMeta=true) a9bde5b3a1b5,41911,1731689306493/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@45687626[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-15T16:48:29,507 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a9bde5b3a1b5,41911,1731689306493, splitWal=true, meta=true 2024-11-15T16:48:29,508 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a9bde5b3a1b5,41911,1731689306493, splitWal=true, meta=true, isMeta: true 2024-11-15T16:48:29,509 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493-splitting 2024-11-15T16:48:29,509 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493-splitting dir is empty, no logs to split. 2024-11-15T16:48:29,509 INFO [PEWorker-2 {}] master.SplitWALManager(105): a9bde5b3a1b5,41911,1731689306493 WAL count=0, meta=true 2024-11-15T16:48:29,512 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493-splitting dir is empty, no logs to split. 2024-11-15T16:48:29,512 INFO [PEWorker-2 {}] master.SplitWALManager(105): a9bde5b3a1b5,41911,1731689306493 WAL count=0, meta=true 2024-11-15T16:48:29,512 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a9bde5b3a1b5,41911,1731689306493 WAL splitting is done? wals=0, meta=true 2024-11-15T16:48:29,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T16:48:29,515 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:48:29,516 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T16:48:29,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:29,598 INFO [RS:0;a9bde5b3a1b5:41911 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:48:29,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41911-0x101acd54e9b0001, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:29,598 INFO [RS:0;a9bde5b3a1b5:41911 {}] regionserver.HRegionServer(1031): Exiting; stopping=a9bde5b3a1b5,41911,1731689306493; zookeeper connection closed. 2024-11-15T16:48:29,598 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a838440 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a838440 2024-11-15T16:48:29,666 DEBUG [a9bde5b3a1b5:39189 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T16:48:29,667 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:29,668 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,41733,1731689306530, state=OPENING 2024-11-15T16:48:29,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:29,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:29,670 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:48:29,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:29,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,41733,1731689306530}] 2024-11-15T16:48:29,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:29,792 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:48:29,793 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:29,793 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:29,793 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:48:29,793 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:48:29,793 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:48:29,793 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T16:48:29,793 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:48:29,794 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36785 2024-11-15T16:48:29,796 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36785 connecting to ZooKeeper ensemble=127.0.0.1:60559 2024-11-15T16:48:29,796 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:29,796 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:29,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367850x0, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:48:29,800 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36785-0x101acd54e9b0003 connected 2024-11-15T16:48:29,800 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-15T16:48:29,800 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-15T16:48:29,801 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T16:48:29,802 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.cache 2024-11-15T16:48:29,802 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T16:48:29,802 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T16:48:29,802 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T16:48:29,802 INFO [Thread-208 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T16:48:29,802 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T16:48:29,802 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T16:48:29,803 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T16:48:29,803 INFO [Thread-208 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-15T16:48:29,805 INFO [Thread-208 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-15T16:48:29,805 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:48:29,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36785 2024-11-15T16:48:29,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36785 2024-11-15T16:48:29,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36785 2024-11-15T16:48:29,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36785 2024-11-15T16:48:29,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36785 2024-11-15T16:48:29,808 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(746): ClusterId : 9ae5ac53-b7f5-40a5-b07b-9a331e5f2620 2024-11-15T16:48:29,808 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T16:48:29,811 DEBUG [Thread-208 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-15T16:48:29,811 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T16:48:29,811 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T16:48:29,811 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T16:48:29,811 INFO [Thread-208 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-15T16:48:29,812 INFO [Thread-208 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.persistence is complete. 2024-11-15T16:48:29,814 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T16:48:29,815 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b6c5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:48:29,823 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:48:29,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42359, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:48:29,827 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T16:48:29,827 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:29,828 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T16:48:29,829 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a9bde5b3a1b5%2C41733%2C1731689306530.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41733,1731689306530, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs, maxLogs=55 2024-11-15T16:48:29,829 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a9bde5b3a1b5:36785 2024-11-15T16:48:29,829 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T16:48:29,829 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T16:48:29,829 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T16:48:29,830 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,39189,1731689306435 with port=36785, startcode=1731689309792 2024-11-15T16:48:29,831 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T16:48:29,833 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50865, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T16:48:29,834 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:29,834 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(517): Registering regionserver=a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:29,836 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913 2024-11-15T16:48:29,836 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T16:48:29,836 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T16:48:29,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:48:29,840 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] zookeeper.ZKUtil(111): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:29,840 WARN [RS:2;a9bde5b3a1b5:36785 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:48:29,840 INFO [RS:2;a9bde5b3a1b5:36785 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:48:29,840 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a9bde5b3a1b5,36785,1731689309792] 2024-11-15T16:48:29,840 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:29,853 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41733,1731689306530/a9bde5b3a1b5%2C41733%2C1731689306530.meta.1731689309829.meta 2024-11-15T16:48:29,853 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:29,854 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:29,854 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T16:48:29,854 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T16:48:29,854 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T16:48:29,854 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T16:48:29,857 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T16:48:29,858 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:29,858 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T16:48:29,858 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T16:48:29,858 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T16:48:29,858 INFO [RS:2;a9bde5b3a1b5:36785 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:48:29,858 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,858 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T16:48:29,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-15T16:48:29,859 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T16:48:29,859 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,859 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,859 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:29,860 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:48:29,861 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/07789c30693b4527bf3c85f6be86ad32, delay=999 ms 2024-11-15T16:48:29,862 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/07789c30693b4527bf3c85f6be86ad32 2024-11-15T16:48:29,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:29,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-15T16:48:29,864 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:48:29,864 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:29,868 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,868 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,868 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,868 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,868 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,868 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,36785,1731689309792-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:48:29,869 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/76d633d74e0a45e29bdc081a0d8f9dcc, delay=963 ms 2024-11-15T16:48:29,871 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/76d633d74e0a45e29bdc081a0d8f9dcc 2024-11-15T16:48:29,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:29,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-15T16:48:29,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:48:29,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:29,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:29,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-15T16:48:29,873 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:48:29,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:29,877 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/8dcfe96637dd42baa22a8837c786e16f, delay=913 ms 2024-11-15T16:48:29,878 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/8dcfe96637dd42baa22a8837c786e16f 2024-11-15T16:48:29,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:48:29,878 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:48:29,879 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:29,879 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740 2024-11-15T16:48:29,880 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:48:29,880 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:48:29,880 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:48:29,881 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:48:29,881 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63227737, jitterRate=-0.05783329904079437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:48:29,882 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T16:48:29,883 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731689309858Writing region info on filesystem at 1731689309858Initializing all the Stores at 1731689309858Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689309859 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689309859Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689309859Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689309859Cleaning up temporary data from old regions at 1731689309880 (+21 ms)Running coprocessor post-open hooks at 1731689309882 (+2 ms)Region opened successfully at 1731689309882 2024-11-15T16:48:29,883 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731689309823 2024-11-15T16:48:29,886 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T16:48:29,886 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T16:48:29,887 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:29,888 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,41733,1731689306530, state=OPEN 2024-11-15T16:48:29,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:29,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:29,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:48:29,891 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:29,891 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:29,891 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:48:29,891 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:29,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-15T16:48:29,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,41733,1731689306530 in 221 msec 2024-11-15T16:48:29,895 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T16:48:29,896 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,36785,1731689309792-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T16:48:29,896 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 381 msec 2024-11-15T16:48:29,896 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.Replication(171): a9bde5b3a1b5,36785,1731689309792 started 2024-11-15T16:48:29,896 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): a9bde5b3a1b5,41911,1731689306493 had 2 regions 2024-11-15T16:48:29,897 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a9bde5b3a1b5,41911,1731689306493, splitWal=true, meta=true, isMeta: false 2024-11-15T16:48:29,898 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493-splitting dir is empty, no logs to split. 2024-11-15T16:48:29,898 INFO [PEWorker-3 {}] master.SplitWALManager(105): a9bde5b3a1b5,41911,1731689306493 WAL count=0, meta=false 2024-11-15T16:48:29,899 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,41911,1731689306493-splitting dir is empty, no logs to split. 2024-11-15T16:48:29,899 INFO [PEWorker-3 {}] master.SplitWALManager(105): a9bde5b3a1b5,41911,1731689306493 WAL count=0, meta=false 2024-11-15T16:48:29,899 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if a9bde5b3a1b5,41911,1731689306493 WAL splitting is done? wals=0, meta=false 2024-11-15T16:48:29,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN}] 2024-11-15T16:48:29,902 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN 2024-11-15T16:48:29,902 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T16:48:29,920 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:48:29,920 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(1482): Serving as a9bde5b3a1b5,36785,1731689309792, RpcServer on a9bde5b3a1b5/172.17.0.2:36785, sessionid=0x101acd54e9b0003 2024-11-15T16:48:29,920 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T16:48:29,921 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:29,921 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,36785,1731689309792' 2024-11-15T16:48:29,921 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T16:48:29,921 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T16:48:29,922 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T16:48:29,922 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T16:48:29,922 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:29,922 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,36785,1731689309792' 2024-11-15T16:48:29,922 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T16:48:29,923 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T16:48:29,923 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T16:48:29,923 INFO [RS:2;a9bde5b3a1b5:36785 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T16:48:29,923 INFO [RS:2;a9bde5b3a1b5:36785 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T16:48:30,024 INFO [RS:2;a9bde5b3a1b5:36785 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:48:30,025 INFO [RS:2;a9bde5b3a1b5:36785 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a9bde5b3a1b5%2C36785%2C1731689309792, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,36785,1731689309792, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs, maxLogs=55 2024-11-15T16:48:30,046 INFO [RS:2;a9bde5b3a1b5:36785 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/WALs/a9bde5b3a1b5,36785,1731689309792/a9bde5b3a1b5%2C36785%2C1731689309792.1731689310026 2024-11-15T16:48:30,046 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T16:48:30,053 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(204): Hosts are {a9bde5b3a1b5=0} racks are {/default-rack=0} 2024-11-15T16:48:30,053 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T16:48:30,053 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T16:48:30,053 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T16:48:30,053 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T16:48:30,054 INFO [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T16:48:30,054 INFO [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T16:48:30,054 DEBUG [a9bde5b3a1b5:39189 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T16:48:30,054 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cd47fcba664f01421cc5a2b65b6cbbce, regionState=OPENING, regionLocation=a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:30,056 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a9bde5b3a1b5:41911 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a9bde5b3a1b5/172.17.0.2:41911 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:48:30,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,41911,1731689306493, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,41911,1731689306493, seqNum=-1, error=java.net.ConnectException: Call to address=a9bde5b3a1b5:41911 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a9bde5b3a1b5/172.17.0.2:41911 2024-11-15T16:48:30,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,41911,1731689306493, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T16:48:30,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,41911,1731689306493, seqNum=-1 from cache 2024-11-15T16:48:30,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address a9bde5b3a1b5:41911 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a9bde5b3a1b5/172.17.0.2:41911 2024-11-15T16:48:30,161 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:48:30,161 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,41733,1731689306530, seqNum=-1] 2024-11-15T16:48:30,161 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:48:30,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33031, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:48:30,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN because future has completed 2024-11-15T16:48:30,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd47fcba664f01421cc5a2b65b6cbbce, server=a9bde5b3a1b5,36785,1731689309792}] 2024-11-15T16:48:30,319 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:48:30,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36207, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:48:30,324 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:30,324 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => cd47fcba664f01421cc5a2b65b6cbbce, NAME => 'testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:48:30,325 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731689307577 cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,325 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:48:30,325 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,325 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,326 INFO [StoreOpener-cd47fcba664f01421cc5a2b65b6cbbce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,327 INFO [StoreOpener-cd47fcba664f01421cc5a2b65b6cbbce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd47fcba664f01421cc5a2b65b6cbbce columnFamilyName family 2024-11-15T16:48:30,327 DEBUG [StoreOpener-cd47fcba664f01421cc5a2b65b6cbbce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:48:30,329 DEBUG [StoreFileOpener-cd47fcba664f01421cc5a2b65b6cbbce-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770, delay=956 ms 2024-11-15T16:48:30,330 DEBUG [StoreOpener-cd47fcba664f01421cc5a2b65b6cbbce-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770 2024-11-15T16:48:30,330 INFO [StoreOpener-cd47fcba664f01421cc5a2b65b6cbbce-1 {}] regionserver.HStore(327): Store=cd47fcba664f01421cc5a2b65b6cbbce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:48:30,331 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,331 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,332 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,332 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,332 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,333 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,333 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened cd47fcba664f01421cc5a2b65b6cbbce; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74278328, jitterRate=0.10683333873748779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:48:30,333 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,334 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for cd47fcba664f01421cc5a2b65b6cbbce: Running coprocessor pre-open hook at 1731689310325Writing region info on filesystem at 1731689310325Initializing all the Stores at 1731689310325Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689310325Cleaning up temporary data from old regions at 1731689310332 (+7 ms)Running coprocessor post-open hooks at 1731689310333 (+1 ms)Region opened successfully at 1731689310334 (+1 ms) 2024-11-15T16:48:30,335 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce., pid=11, masterSystemTime=1731689310319 2024-11-15T16:48:30,337 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:30,338 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:30,338 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cd47fcba664f01421cc5a2b65b6cbbce, regionState=OPEN, openSeqNum=10, regionLocation=a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:30,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd47fcba664f01421cc5a2b65b6cbbce, server=a9bde5b3a1b5,36785,1731689309792 because future has completed 2024-11-15T16:48:30,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-15T16:48:30,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure cd47fcba664f01421cc5a2b65b6cbbce, server=a9bde5b3a1b5,36785,1731689309792 in 175 msec 2024-11-15T16:48:30,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-15T16:48:30,345 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server a9bde5b3a1b5,41911,1731689306493 after splitting done 2024-11-15T16:48:30,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731689307577, region=cd47fcba664f01421cc5a2b65b6cbbce, ASSIGN in 443 msec 2024-11-15T16:48:30,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure a9bde5b3a1b5,41911,1731689306493, splitWal=true, meta=true in 845 msec 2024-11-15T16:48:30,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T16:48:30,421 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:48:30,421 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:48:30,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:30,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:30,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:48:30,422 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T16:48:30,422 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=476740598, stopped=false 2024-11-15T16:48:30,422 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a9bde5b3a1b5,39189,1731689306435 2024-11-15T16:48:30,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:30,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:30,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:30,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:30,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T16:48:30,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:30,426 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T16:48:30,426 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:48:30,426 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:48:30,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:30,427 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a9bde5b3a1b5,41733,1731689306530' ***** 2024-11-15T16:48:30,427 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T16:48:30,427 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a9bde5b3a1b5,36785,1731689309792' ***** 2024-11-15T16:48:30,427 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T16:48:30,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:30,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:30,428 INFO [RS:2;a9bde5b3a1b5:36785 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T16:48:30,428 INFO [RS:2;a9bde5b3a1b5:36785 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.persistence 2024-11-15T16:48:30,428 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T16:48:30,428 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T16:48:30,428 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T16:48:30,428 INFO [RS:2;a9bde5b3a1b5:36785 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T16:48:30,428 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T16:48:30,429 INFO [RS:1;a9bde5b3a1b5:41733 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T16:48:30,429 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T16:48:30,429 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T16:48:30,429 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T16:48:30,429 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:48:30,430 INFO [RS:1;a9bde5b3a1b5:41733 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ecc3954-f427-d104-68b0-d63b726bc8ae/bucket.persistence 2024-11-15T16:48:30,430 INFO [RS:1;a9bde5b3a1b5:41733 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T16:48:30,430 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T16:48:30,438 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] bucket.BucketCache(1487): Thread RS:2;a9bde5b3a1b5:36785 started persisting bucket cache to file 2024-11-15T16:48:30,438 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:30,438 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:30,440 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] bucket.BucketCache(1487): Thread RS:1;a9bde5b3a1b5:41733 started persisting bucket cache to file 2024-11-15T16:48:30,440 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T16:48:30,440 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T16:48:30,445 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-15T16:48:30,445 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731689310438 2024-11-15T16:48:30,445 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] bucket.BucketCache(1509): Thread RS:2;a9bde5b3a1b5:36785 finished persisting bucket cache to file, renaming 2024-11-15T16:48:30,446 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T16:48:30,446 INFO [RS:2;a9bde5b3a1b5:36785 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T16:48:30,446 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T16:48:30,446 INFO [RS:2;a9bde5b3a1b5:36785 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T16:48:30,446 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(3091): Received CLOSE for cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,446 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(959): stopping server a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:30,446 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:48:30,447 INFO [RS:2;a9bde5b3a1b5:36785 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a9bde5b3a1b5:36785. 2024-11-15T16:48:30,447 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T16:48:30,447 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:48:30,447 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:30,447 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cd47fcba664f01421cc5a2b65b6cbbce, disabling compactions & flushes 2024-11-15T16:48:30,447 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:30,447 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T16:48:30,447 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:30,447 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(1325): Online Regions={cd47fcba664f01421cc5a2b65b6cbbce=testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce.} 2024-11-15T16:48:30,447 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. after waiting 0 ms 2024-11-15T16:48:30,447 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(1351): Waiting on cd47fcba664f01421cc5a2b65b6cbbce 2024-11-15T16:48:30,447 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:30,448 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T16:48:30,448 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731689310440 2024-11-15T16:48:30,448 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] bucket.BucketCache(1509): Thread RS:1;a9bde5b3a1b5:41733 finished persisting bucket cache to file, renaming 2024-11-15T16:48:30,448 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T16:48:30,448 INFO [RS:1;a9bde5b3a1b5:41733 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T16:48:30,448 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T16:48:30,449 INFO [RS:1;a9bde5b3a1b5:41733 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T16:48:30,449 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(959): stopping server a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:30,449 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:48:30,449 INFO [RS:1;a9bde5b3a1b5:41733 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a9bde5b3a1b5:41733. 2024-11-15T16:48:30,449 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:48:30,449 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:30,449 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T16:48:30,449 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T16:48:30,449 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T16:48:30,449 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T16:48:30,454 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T16:48:30,454 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T16:48:30,455 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T16:48:30,455 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T16:48:30,455 DEBUG [StoreFileCloser-cd47fcba664f01421cc5a2b65b6cbbce-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/family/1e182b5c0589431ebc3bfcf641de6770 2024-11-15T16:48:30,455 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T16:48:30,455 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T16:48:30,455 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T16:48:30,455 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T16:48:30,455 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T16:48:30,455 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-15T16:48:30,461 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/default/testBlockEvictionOnGracefulStop1731689307577/cd47fcba664f01421cc5a2b65b6cbbce/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-15T16:48:30,461 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:30,462 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cd47fcba664f01421cc5a2b65b6cbbce: Waiting for close lock at 1731689310447Running coprocessor pre-close hooks at 1731689310447Disabling compacts and flushes for region at 1731689310447Disabling writes for close at 1731689310447Writing region close event to WAL at 1731689310455 (+8 ms)Running coprocessor post-close hooks at 1731689310461 (+6 ms)Closed at 1731689310461 2024-11-15T16:48:30,462 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce. 2024-11-15T16:48:30,469 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ac802e46932b4d34814b17aa6247a94c 2024-11-15T16:48:30,471 INFO [regionserver/a9bde5b3a1b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:30,484 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/info/ac802e46932b4d34814b17aa6247a94c is 209, key is testBlockEvictionOnGracefulStop1731689307577,,1731689307582.cd47fcba664f01421cc5a2b65b6cbbce./info:regioninfo/1731689310338/Put/seqid=0 2024-11-15T16:48:30,484 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/info/ac802e46932b4d34814b17aa6247a94c 2024-11-15T16:48:30,486 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/.tmp/info/ac802e46932b4d34814b17aa6247a94c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/ac802e46932b4d34814b17aa6247a94c 2024-11-15T16:48:30,487 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/ac802e46932b4d34814b17aa6247a94c, delay=999 ms 2024-11-15T16:48:30,488 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/ac802e46932b4d34814b17aa6247a94c, entries=8, sequenceid=19, filesize=6.6 K 2024-11-15T16:48:30,489 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=19, compaction requested=false 2024-11-15T16:48:30,490 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/07789c30693b4527bf3c85f6be86ad32 2024-11-15T16:48:30,490 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/info/ac802e46932b4d34814b17aa6247a94c 2024-11-15T16:48:30,491 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/ns/76d633d74e0a45e29bdc081a0d8f9dcc 2024-11-15T16:48:30,491 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/table/8dcfe96637dd42baa22a8837c786e16f 2024-11-15T16:48:30,498 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-15T16:48:30,498 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T16:48:30,498 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T16:48:30,499 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731689310455Running coprocessor pre-close hooks at 1731689310455Disabling compacts and flushes for region at 1731689310455Disabling writes for close at 1731689310455Obtaining lock to block concurrent updates at 1731689310455Preparing flush snapshotting stores in 1588230740 at 1731689310455Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731689310456 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731689310456Flushing 1588230740/info: creating writer at 1731689310456Flushing 1588230740/info: appending metadata at 1731689310483 (+27 ms)Flushing 1588230740/info: closing flushed file at 1731689310483Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fe351e: reopening flushed file at 1731689310486 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=19, compaction requested=false at 1731689310489 (+3 ms)Writing region close event to WAL at 1731689310492 (+3 ms)Running coprocessor post-close hooks at 1731689310498 (+6 ms)Closed at 1731689310498 2024-11-15T16:48:30,499 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T16:48:30,519 INFO [regionserver/a9bde5b3a1b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:30,647 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(976): stopping server a9bde5b3a1b5,36785,1731689309792; all regions closed. 2024-11-15T16:48:30,649 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs 2024-11-15T16:48:30,650 INFO [RS:2;a9bde5b3a1b5:36785 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C36785%2C1731689309792:(num 1731689310026) 2024-11-15T16:48:30,650 DEBUG [RS:2;a9bde5b3a1b5:36785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:30,650 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:30,650 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:48:30,650 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.ChoreService(370): Chore service for: regionserver/a9bde5b3a1b5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T16:48:30,650 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T16:48:30,650 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T16:48:30,650 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T16:48:30,650 INFO [regionserver/a9bde5b3a1b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:48:30,650 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:48:30,650 INFO [RS:2;a9bde5b3a1b5:36785 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36785 2024-11-15T16:48:30,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a9bde5b3a1b5,36785,1731689309792 2024-11-15T16:48:30,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:48:30,653 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:48:30,655 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(976): stopping server a9bde5b3a1b5,41733,1731689306530; all regions closed. 2024-11-15T16:48:30,655 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a9bde5b3a1b5,36785,1731689309792] 2024-11-15T16:48:30,656 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs 2024-11-15T16:48:30,656 INFO [RS:1;a9bde5b3a1b5:41733 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C41733%2C1731689306530.meta:.meta(num 1731689309829) 2024-11-15T16:48:30,658 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/oldWALs 2024-11-15T16:48:30,658 INFO [RS:1;a9bde5b3a1b5:41733 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C41733%2C1731689306530:(num 1731689307061) 2024-11-15T16:48:30,658 DEBUG [RS:1;a9bde5b3a1b5:41733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:48:30,658 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a9bde5b3a1b5,36785,1731689309792 already deleted, retry=false 2024-11-15T16:48:30,658 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:48:30,658 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a9bde5b3a1b5,36785,1731689309792 expired; onlineServers=1 2024-11-15T16:48:30,658 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:48:30,659 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.ChoreService(370): Chore service for: regionserver/a9bde5b3a1b5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T16:48:30,659 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:48:30,659 INFO [regionserver/a9bde5b3a1b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:48:30,659 INFO [RS:1;a9bde5b3a1b5:41733 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41733 2024-11-15T16:48:30,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:48:30,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a9bde5b3a1b5,41733,1731689306530 2024-11-15T16:48:30,661 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:48:30,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a9bde5b3a1b5,41733,1731689306530] 2024-11-15T16:48:30,664 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a9bde5b3a1b5,41733,1731689306530 already deleted, retry=false 2024-11-15T16:48:30,665 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a9bde5b3a1b5,41733,1731689306530 expired; onlineServers=0 2024-11-15T16:48:30,665 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a9bde5b3a1b5,39189,1731689306435' ***** 2024-11-15T16:48:30,665 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T16:48:30,665 INFO [M:0;a9bde5b3a1b5:39189 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:48:30,665 INFO [M:0;a9bde5b3a1b5:39189 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:48:30,665 DEBUG [M:0;a9bde5b3a1b5:39189 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T16:48:30,665 DEBUG [M:0;a9bde5b3a1b5:39189 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T16:48:30,665 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T16:48:30,665 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689306825 {}] cleaner.HFileCleaner(306): Exit Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689306825,5,FailOnTimeoutGroup] 2024-11-15T16:48:30,665 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689306825 {}] cleaner.HFileCleaner(306): Exit Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689306825,5,FailOnTimeoutGroup] 2024-11-15T16:48:30,665 INFO [M:0;a9bde5b3a1b5:39189 {}] hbase.ChoreService(370): Chore service for: master/a9bde5b3a1b5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T16:48:30,666 INFO [M:0;a9bde5b3a1b5:39189 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:48:30,666 DEBUG [M:0;a9bde5b3a1b5:39189 {}] master.HMaster(1795): Stopping service threads 2024-11-15T16:48:30,666 INFO [M:0;a9bde5b3a1b5:39189 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T16:48:30,666 INFO [M:0;a9bde5b3a1b5:39189 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T16:48:30,666 INFO [M:0;a9bde5b3a1b5:39189 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T16:48:30,666 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T16:48:30,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T16:48:30,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:48:30,667 DEBUG [M:0;a9bde5b3a1b5:39189 {}] zookeeper.ZKUtil(347): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T16:48:30,667 WARN [M:0;a9bde5b3a1b5:39189 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T16:48:30,667 INFO [M:0;a9bde5b3a1b5:39189 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/.lastflushedseqids 2024-11-15T16:48:30,671 INFO [M:0;a9bde5b3a1b5:39189 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T16:48:30,671 INFO [M:0;a9bde5b3a1b5:39189 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T16:48:30,672 DEBUG [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T16:48:30,672 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:30,672 DEBUG [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:30,672 DEBUG [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T16:48:30,672 DEBUG [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:30,672 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-15T16:48:30,682 DEBUG [M:0;a9bde5b3a1b5:39189 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5d5d610a8c9e45a29d6aa8ffc839467d 2024-11-15T16:48:30,697 DEBUG [M:0;a9bde5b3a1b5:39189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d5d610a8c9e45a29d6aa8ffc839467d is 82, key is hbase:meta,,1/info:regioninfo/1731689309886/Put/seqid=0 2024-11-15T16:48:30,697 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d5d610a8c9e45a29d6aa8ffc839467d 2024-11-15T16:48:30,706 DEBUG [M:0;a9bde5b3a1b5:39189 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5a13b540ad92441a91cf6267e8819f0a 2024-11-15T16:48:30,721 DEBUG [M:0;a9bde5b3a1b5:39189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5a13b540ad92441a91cf6267e8819f0a is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731689307965/Put/seqid=0 2024-11-15T16:48:30,721 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5a13b540ad92441a91cf6267e8819f0a 2024-11-15T16:48:30,730 DEBUG [M:0;a9bde5b3a1b5:39189 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8341e5e385ae4f2baee881343498bc8a 2024-11-15T16:48:30,746 DEBUG [M:0;a9bde5b3a1b5:39189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8341e5e385ae4f2baee881343498bc8a is 69, key is a9bde5b3a1b5,36785,1731689309792/rs:state/1731689309835/Put/seqid=0 2024-11-15T16:48:30,747 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8341e5e385ae4f2baee881343498bc8a 2024-11-15T16:48:30,748 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8341e5e385ae4f2baee881343498bc8a 2024-11-15T16:48:30,749 DEBUG [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d5d610a8c9e45a29d6aa8ffc839467d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d5d610a8c9e45a29d6aa8ffc839467d 2024-11-15T16:48:30,751 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d5d610a8c9e45a29d6aa8ffc839467d, entries=14, sequenceid=102, filesize=5.9 K 2024-11-15T16:48:30,751 DEBUG [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5a13b540ad92441a91cf6267e8819f0a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5a13b540ad92441a91cf6267e8819f0a 2024-11-15T16:48:30,753 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5a13b540ad92441a91cf6267e8819f0a, entries=11, sequenceid=102, filesize=6.5 K 2024-11-15T16:48:30,753 DEBUG [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8341e5e385ae4f2baee881343498bc8a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8341e5e385ae4f2baee881343498bc8a 2024-11-15T16:48:30,754 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8341e5e385ae4f2baee881343498bc8a 2024-11-15T16:48:30,755 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/275aba1f-6368-d5bf-2b4c-d66609d66d0a/d4301b86-fbec-95c8-2417-d01f7dc80913/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8341e5e385ae4f2baee881343498bc8a, entries=3, sequenceid=102, filesize=5.3 K 2024-11-15T16:48:30,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:30,755 INFO [RS:2;a9bde5b3a1b5:36785 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:48:30,756 INFO [RS:2;a9bde5b3a1b5:36785 {}] regionserver.HRegionServer(1031): Exiting; stopping=a9bde5b3a1b5,36785,1731689309792; zookeeper connection closed. 2024-11-15T16:48:30,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36785-0x101acd54e9b0003, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:30,756 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=102, compaction requested=false 2024-11-15T16:48:30,756 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4fa0bc6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4fa0bc6 2024-11-15T16:48:30,758 INFO [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:48:30,758 DEBUG [M:0;a9bde5b3a1b5:39189 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731689310671Disabling compacts and flushes for region at 1731689310671Disabling writes for close at 1731689310672 (+1 ms)Obtaining lock to block concurrent updates at 1731689310672Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731689310672Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1731689310673 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731689310673Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731689310673Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731689310696 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731689310696Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731689310699 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731689310720 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731689310720Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731689310723 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731689310745 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731689310745Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e031220: reopening flushed file at 1731689310749 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65de24ee: reopening flushed file at 1731689310751 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d6a3a25: reopening flushed file at 1731689310753 (+2 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=102, compaction requested=false at 1731689310756 (+3 ms)Writing region close event to WAL at 1731689310758 (+2 ms)Closed at 1731689310758 2024-11-15T16:48:30,759 INFO [M:0;a9bde5b3a1b5:39189 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T16:48:30,759 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:48:30,759 INFO [M:0;a9bde5b3a1b5:39189 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39189 2024-11-15T16:48:30,760 INFO [M:0;a9bde5b3a1b5:39189 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:48:30,763 INFO [RS:1;a9bde5b3a1b5:41733 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:48:30,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:30,763 INFO [RS:1;a9bde5b3a1b5:41733 {}] regionserver.HRegionServer(1031): Exiting; stopping=a9bde5b3a1b5,41733,1731689306530; zookeeper connection closed. 2024-11-15T16:48:30,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x101acd54e9b0002, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:30,764 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2dfb20c5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2dfb20c5 2024-11-15T16:48:30,764 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-15T16:48:30,862 INFO [M:0;a9bde5b3a1b5:39189 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:48:30,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:30,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101acd54e9b0000, quorum=127.0.0.1:60559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:48:30,866 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T16:48:30,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-15T16:48:30,869 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:48:30,876 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1152 (was 1235), ProcessCount=11 (was 11), AvailableMemoryMB=6025 (was 6120)