2024-12-08 21:02:00,175 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-08 21:02:00,185 main DEBUG Took 0.008654 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 21:02:00,185 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 21:02:00,186 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 21:02:00,192 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 21:02:00,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,201 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 21:02:00,215 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,217 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,218 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,220 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,221 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,222 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,223 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,223 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,224 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,224 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,224 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,225 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,226 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,226 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,227 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,227 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,227 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,228 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:00,228 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,229 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 21:02:00,230 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:00,232 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 21:02:00,233 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 21:02:00,234 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 21:02:00,236 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 21:02:00,237 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 21:02:00,246 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 21:02:00,249 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 21:02:00,250 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 21:02:00,251 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 21:02:00,251 main DEBUG createAppenders(={Console}) 2024-12-08 21:02:00,252 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-08 21:02:00,252 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-08 21:02:00,253 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-08 21:02:00,253 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 21:02:00,254 main DEBUG OutputStream closed 2024-12-08 21:02:00,254 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 21:02:00,254 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 21:02:00,255 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-08 21:02:00,330 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 21:02:00,332 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 21:02:00,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 21:02:00,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 21:02:00,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 21:02:00,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 21:02:00,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 21:02:00,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 21:02:00,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 21:02:00,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 21:02:00,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 21:02:00,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 21:02:00,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 21:02:00,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 21:02:00,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 21:02:00,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 21:02:00,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 21:02:00,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 21:02:00,341 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 21:02:00,341 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-08 21:02:00,341 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 21:02:00,342 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-08T21:02:00,353 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-08 21:02:00,356 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 21:02:00,356 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T21:02:00,555 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e 2024-12-08T21:02:00,577 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=407, ProcessCount=11, AvailableMemoryMB=17253 2024-12-08T21:02:00,650 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T21:02:00,723 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:00,736 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/cluster_278e32be-c4be-640c-9cba-d72afd07b0ef, deleteOnExit=true 2024-12-08T21:02:00,873 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/cluster_278e32be-c4be-640c-9cba-d72afd07b0ef/zookeeper_0, clientPort=52417, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/cluster_278e32be-c4be-640c-9cba-d72afd07b0ef/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/cluster_278e32be-c4be-640c-9cba-d72afd07b0ef/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T21:02:00,884 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52417 2024-12-08T21:02:00,884 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:00,963 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 with version=8 2024-12-08T21:02:00,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/hbase-staging 2024-12-08T21:02:01,048 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T21:02:01,264 INFO [Time-limited test {}] client.ConnectionUtils(128): master/229bab1f9d30:0 server-side Connection retries=45 2024-12-08T21:02:01,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:01,275 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:01,284 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:02:01,285 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:01,285 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:02:01,409 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T21:02:01,482 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:02:01,515 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T21:02:01,524 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T21:02:01,528 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:02:01,560 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 111027 (auto-detected) 2024-12-08T21:02:01,561 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T21:02:01,587 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37617 2024-12-08T21:02:01,608 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37617 connecting to ZooKeeper ensemble=127.0.0.1:52417 2024-12-08T21:02:01,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:376170x0, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:02:01,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37617-0x100074a03310000 connected 2024-12-08T21:02:01,825 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:01,826 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:01,838 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:01,841 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200, hbase.cluster.distributed=false 2024-12-08T21:02:01,863 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:02:01,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37617 2024-12-08T21:02:01,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37617 2024-12-08T21:02:01,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37617 2024-12-08T21:02:01,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37617 2024-12-08T21:02:01,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37617 2024-12-08T21:02:01,937 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/229bab1f9d30:0 server-side Connection retries=45 2024-12-08T21:02:01,938 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:01,938 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:01,939 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:02:01,939 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:01,939 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:02:01,941 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T21:02:01,943 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:02:01,944 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45313 2024-12-08T21:02:01,946 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45313 connecting to ZooKeeper ensemble=127.0.0.1:52417 2024-12-08T21:02:01,946 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:01,947 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:01,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453130x0, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:02:01,979 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:01,979 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45313-0x100074a03310001 connected 2024-12-08T21:02:01,987 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T21:02:02,001 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.cache 2024-12-08T21:02:02,006 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T21:02:02,007 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T21:02:02,008 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T21:02:02,010 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T21:02:02,010 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-08T21:02:02,015 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T21:02:02,017 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T21:02:02,017 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.persistence is complete. 2024-12-08T21:02:02,019 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T21:02:02,022 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T21:02:02,026 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:02:02,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45313 2024-12-08T21:02:02,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45313 2024-12-08T21:02:02,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45313 2024-12-08T21:02:02,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45313 2024-12-08T21:02:02,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45313 2024-12-08T21:02:02,047 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/229bab1f9d30:0 server-side Connection retries=45 2024-12-08T21:02:02,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:02,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:02,047 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:02:02,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:02,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:02:02,048 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T21:02:02,048 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:02:02,049 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41705 2024-12-08T21:02:02,051 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41705 connecting to ZooKeeper ensemble=127.0.0.1:52417 2024-12-08T21:02:02,051 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:02,052 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:02,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417050x0, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:02:02,088 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417050x0, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:02,088 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41705-0x100074a03310002 connected 2024-12-08T21:02:02,089 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T21:02:02,090 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.cache 2024-12-08T21:02:02,090 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T21:02:02,090 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T21:02:02,090 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T21:02:02,091 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T21:02:02,091 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-08T21:02:02,092 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T21:02:02,092 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T21:02:02,092 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T21:02:02,092 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.persistence is complete. 2024-12-08T21:02:02,093 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T21:02:02,095 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:02:02,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41705 2024-12-08T21:02:02,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41705 2024-12-08T21:02:02,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41705 2024-12-08T21:02:02,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41705 2024-12-08T21:02:02,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41705 2024-12-08T21:02:02,114 DEBUG [M:0;229bab1f9d30:37617 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;229bab1f9d30:37617 2024-12-08T21:02:02,115 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/229bab1f9d30,37617,1733691721113 2024-12-08T21:02:02,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:02,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:02,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:02,131 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/229bab1f9d30,37617,1733691721113 2024-12-08T21:02:02,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T21:02:02,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T21:02:02,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,163 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T21:02:02,164 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/229bab1f9d30,37617,1733691721113 from backup master directory 2024-12-08T21:02:02,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:02,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/229bab1f9d30,37617,1733691721113 2024-12-08T21:02:02,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:02,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:02,171 WARN [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:02:02,171 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=229bab1f9d30,37617,1733691721113 2024-12-08T21:02:02,173 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T21:02:02,174 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T21:02:02,183 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/hbase.id] with ID: c36c25c1-e421-4ea6-be64-25b589071f61 2024-12-08T21:02:02,183 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/.tmp/hbase.id 2024-12-08T21:02:02,188 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/hbase.id] 2024-12-08T21:02:02,222 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:02,227 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T21:02:02,245 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-08T21:02:02,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,301 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:02:02,304 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T21:02:02,340 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:02:02,346 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:02,405 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store 2024-12-08T21:02:02,423 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T21:02:02,426 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:02,427 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T21:02:02,427 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:02,427 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:02,429 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T21:02:02,429 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:02,429 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:02,430 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733691722427Disabling compacts and flushes for region at 1733691722427Disabling writes for close at 1733691722429 (+2 ms)Writing region close event to WAL at 1733691722429Closed at 1733691722429 2024-12-08T21:02:02,430 WARN [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/.initializing 2024-12-08T21:02:02,431 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/WALs/229bab1f9d30,37617,1733691721113 2024-12-08T21:02:02,441 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:02:02,453 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=229bab1f9d30%2C37617%2C1733691721113, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/WALs/229bab1f9d30,37617,1733691721113, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/oldWALs, maxLogs=10 2024-12-08T21:02:02,485 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/WALs/229bab1f9d30,37617,1733691721113/229bab1f9d30%2C37617%2C1733691721113.1733691722457 2024-12-08T21:02:02,486 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T21:02:02,486 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:02,487 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:02,490 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,491 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T21:02:02,554 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:02,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:02,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T21:02:02,561 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:02,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:02,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,566 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T21:02:02,566 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:02,566 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:02,567 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,571 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T21:02:02,572 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:02,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:02,573 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,576 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,576 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,579 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,579 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,582 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:02:02,584 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:02,589 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:02:02,590 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70196243, jitterRate=0.04600553214550018}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:02:02,596 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733691722502Initializing all the Stores at 1733691722503 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691722503Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691722505 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691722505Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691722505Cleaning up temporary data from old regions at 1733691722579 (+74 ms)Region opened successfully at 1733691722596 (+17 ms) 2024-12-08T21:02:02,598 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T21:02:02,631 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1af41e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=229bab1f9d30/172.17.0.2:0 2024-12-08T21:02:02,660 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T21:02:02,672 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T21:02:02,672 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T21:02:02,675 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T21:02:02,676 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-08T21:02:02,681 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-08T21:02:02,682 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T21:02:02,712 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T21:02:02,722 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T21:02:02,770 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T21:02:02,773 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T21:02:02,775 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T21:02:02,786 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T21:02:02,789 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T21:02:02,792 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T21:02:02,803 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T21:02:02,805 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T21:02:02,811 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T21:02:02,830 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T21:02:02,836 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T21:02:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,856 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=229bab1f9d30,37617,1733691721113, sessionid=0x100074a03310000, setting cluster-up flag (Was=false) 2024-12-08T21:02:02,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,912 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T21:02:02,915 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=229bab1f9d30,37617,1733691721113 2024-12-08T21:02:02,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:02,961 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T21:02:02,963 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=229bab1f9d30,37617,1733691721113 2024-12-08T21:02:02,969 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T21:02:03,004 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(746): ClusterId : c36c25c1-e421-4ea6-be64-25b589071f61 2024-12-08T21:02:03,004 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(746): ClusterId : c36c25c1-e421-4ea6-be64-25b589071f61 2024-12-08T21:02:03,006 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T21:02:03,006 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T21:02:03,021 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T21:02:03,021 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T21:02:03,021 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T21:02:03,021 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T21:02:03,030 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T21:02:03,030 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T21:02:03,030 DEBUG [RS:1;229bab1f9d30:41705 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfeacc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=229bab1f9d30/172.17.0.2:0 2024-12-08T21:02:03,030 DEBUG [RS:0;229bab1f9d30:45313 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335f1a55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=229bab1f9d30/172.17.0.2:0 2024-12-08T21:02:03,037 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:03,043 DEBUG [RS:0;229bab1f9d30:45313 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;229bab1f9d30:45313 2024-12-08T21:02:03,047 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T21:02:03,047 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T21:02:03,047 DEBUG [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T21:02:03,050 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(2659): reportForDuty to master=229bab1f9d30,37617,1733691721113 with port=45313, startcode=1733691721907 2024-12-08T21:02:03,049 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T21:02:03,050 DEBUG [RS:1;229bab1f9d30:41705 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;229bab1f9d30:41705 2024-12-08T21:02:03,050 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T21:02:03,050 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T21:02:03,051 DEBUG [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T21:02:03,052 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(2659): reportForDuty to master=229bab1f9d30,37617,1733691721113 with port=41705, startcode=1733691722046 2024-12-08T21:02:03,058 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T21:02:03,061 DEBUG [RS:1;229bab1f9d30:41705 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T21:02:03,061 DEBUG [RS:0;229bab1f9d30:45313 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T21:02:03,064 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 229bab1f9d30,37617,1733691721113 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T21:02:03,071 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:03,072 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:03,072 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:03,072 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:03,072 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/229bab1f9d30:0, corePoolSize=10, maxPoolSize=10 2024-12-08T21:02:03,072 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,073 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/229bab1f9d30:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:02:03,073 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,074 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733691753074 2024-12-08T21:02:03,076 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T21:02:03,077 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:03,077 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T21:02:03,078 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T21:02:03,081 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T21:02:03,081 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T21:02:03,082 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T21:02:03,082 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T21:02:03,083 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:03,083 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T21:02:03,083 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,087 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T21:02:03,088 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T21:02:03,089 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T21:02:03,091 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T21:02:03,092 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T21:02:03,099 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.large.0-1733691723093,5,FailOnTimeoutGroup] 2024-12-08T21:02:03,100 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.small.0-1733691723099,5,FailOnTimeoutGroup] 2024-12-08T21:02:03,100 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,100 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T21:02:03,101 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,101 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,116 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T21:02:03,116 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 2024-12-08T21:02:03,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:03,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T21:02:03,135 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T21:02:03,137 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51917, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T21:02:03,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T21:02:03,138 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:03,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:03,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T21:02:03,141 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37617 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 229bab1f9d30,45313,1733691721907 2024-12-08T21:02:03,142 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T21:02:03,142 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:03,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:03,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T21:02:03,144 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37617 {}] master.ServerManager(517): Registering regionserver=229bab1f9d30,45313,1733691721907 2024-12-08T21:02:03,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T21:02:03,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:03,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:03,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T21:02:03,152 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T21:02:03,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:03,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:03,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T21:02:03,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740 2024-12-08T21:02:03,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740 2024-12-08T21:02:03,154 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37617 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 229bab1f9d30,41705,1733691722046 2024-12-08T21:02:03,155 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37617 {}] master.ServerManager(517): Registering regionserver=229bab1f9d30,41705,1733691722046 2024-12-08T21:02:03,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T21:02:03,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T21:02:03,155 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:02:03,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T21:02:03,159 DEBUG [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 2024-12-08T21:02:03,159 DEBUG [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T21:02:03,159 DEBUG [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T21:02:03,160 DEBUG [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 2024-12-08T21:02:03,160 DEBUG [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T21:02:03,160 DEBUG [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T21:02:03,165 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:02:03,165 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67499174, jitterRate=0.005816072225570679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:02:03,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733691723129Initializing all the Stores at 1733691723130 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691723130Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691723131 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691723131Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691723131Cleaning up temporary data from old regions at 1733691723155 (+24 ms)Region opened successfully at 1733691723168 (+13 ms) 2024-12-08T21:02:03,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T21:02:03,168 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T21:02:03,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T21:02:03,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T21:02:03,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T21:02:03,170 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T21:02:03,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733691723168Disabling compacts and flushes for region at 1733691723168Disabling writes for close at 1733691723168Writing region close event to WAL at 1733691723169 (+1 ms)Closed at 1733691723170 (+1 ms) 2024-12-08T21:02:03,171 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:03,171 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T21:02:03,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T21:02:03,185 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T21:02:03,187 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T21:02:03,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:02:03,195 DEBUG [RS:1;229bab1f9d30:41705 {}] zookeeper.ZKUtil(111): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/229bab1f9d30,41705,1733691722046 2024-12-08T21:02:03,195 DEBUG [RS:0;229bab1f9d30:45313 {}] zookeeper.ZKUtil(111): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/229bab1f9d30,45313,1733691721907 2024-12-08T21:02:03,195 WARN [RS:0;229bab1f9d30:45313 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:02:03,195 WARN [RS:1;229bab1f9d30:41705 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:02:03,196 INFO [RS:0;229bab1f9d30:45313 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:03,196 INFO [RS:1;229bab1f9d30:41705 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:03,196 DEBUG [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,41705,1733691722046 2024-12-08T21:02:03,196 DEBUG [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,45313,1733691721907 2024-12-08T21:02:03,198 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [229bab1f9d30,45313,1733691721907] 2024-12-08T21:02:03,198 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [229bab1f9d30,41705,1733691722046] 2024-12-08T21:02:03,218 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T21:02:03,218 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T21:02:03,235 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T21:02:03,235 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T21:02:03,240 INFO [RS:1;229bab1f9d30:41705 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T21:02:03,240 INFO [RS:0;229bab1f9d30:45313 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T21:02:03,240 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,240 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,241 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T21:02:03,241 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T21:02:03,246 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T21:02:03,246 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T21:02:03,247 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,247 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,248 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/229bab1f9d30:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:02:03,248 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,248 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/229bab1f9d30:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:02:03,249 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,249 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,249 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,249 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,249 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,249 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,249 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,249 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,249 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:03,249 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,249 DEBUG [RS:1;229bab1f9d30:41705 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:03,249 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,250 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:03,250 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:03,250 DEBUG [RS:0;229bab1f9d30:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:03,255 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,255 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,255 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,255 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,256 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,256 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,256 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,256 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,256 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,256 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,256 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,45313,1733691721907-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:02:03,256 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,41705,1733691722046-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:02:03,274 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T21:02:03,274 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T21:02:03,276 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,45313,1733691721907-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,276 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,41705,1733691722046-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,276 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,276 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,276 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.Replication(171): 229bab1f9d30,41705,1733691722046 started 2024-12-08T21:02:03,276 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.Replication(171): 229bab1f9d30,45313,1733691721907 started 2024-12-08T21:02:03,293 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,293 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:03,294 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(1482): Serving as 229bab1f9d30,41705,1733691722046, RpcServer on 229bab1f9d30/172.17.0.2:41705, sessionid=0x100074a03310002 2024-12-08T21:02:03,294 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(1482): Serving as 229bab1f9d30,45313,1733691721907, RpcServer on 229bab1f9d30/172.17.0.2:45313, sessionid=0x100074a03310001 2024-12-08T21:02:03,295 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T21:02:03,295 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T21:02:03,295 DEBUG [RS:1;229bab1f9d30:41705 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 229bab1f9d30,41705,1733691722046 2024-12-08T21:02:03,295 DEBUG [RS:0;229bab1f9d30:45313 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 229bab1f9d30,45313,1733691721907 2024-12-08T21:02:03,295 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,41705,1733691722046' 2024-12-08T21:02:03,295 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,45313,1733691721907' 2024-12-08T21:02:03,295 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T21:02:03,295 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T21:02:03,296 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T21:02:03,296 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T21:02:03,297 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T21:02:03,297 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T21:02:03,297 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T21:02:03,297 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T21:02:03,297 DEBUG [RS:1;229bab1f9d30:41705 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 229bab1f9d30,41705,1733691722046 2024-12-08T21:02:03,297 DEBUG [RS:0;229bab1f9d30:45313 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 229bab1f9d30,45313,1733691721907 2024-12-08T21:02:03,297 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,41705,1733691722046' 2024-12-08T21:02:03,297 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,45313,1733691721907' 2024-12-08T21:02:03,297 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T21:02:03,297 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T21:02:03,298 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T21:02:03,298 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T21:02:03,298 DEBUG [RS:1;229bab1f9d30:41705 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T21:02:03,298 DEBUG [RS:0;229bab1f9d30:45313 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T21:02:03,298 INFO [RS:0;229bab1f9d30:45313 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T21:02:03,298 INFO [RS:1;229bab1f9d30:41705 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T21:02:03,298 INFO [RS:0;229bab1f9d30:45313 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T21:02:03,298 INFO [RS:1;229bab1f9d30:41705 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T21:02:03,338 WARN [229bab1f9d30:37617 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:03,407 INFO [RS:0;229bab1f9d30:45313 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:02:03,407 INFO [RS:1;229bab1f9d30:41705 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:02:03,408 INFO [RS:0;229bab1f9d30:45313 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=229bab1f9d30%2C45313%2C1733691721907, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,45313,1733691721907, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs, maxLogs=55 2024-12-08T21:02:03,408 INFO [RS:1;229bab1f9d30:41705 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=229bab1f9d30%2C41705%2C1733691722046, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,41705,1733691722046, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs, maxLogs=55 2024-12-08T21:02:03,428 INFO [RS:0;229bab1f9d30:45313 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,45313,1733691721907/229bab1f9d30%2C45313%2C1733691721907.1733691723411 2024-12-08T21:02:03,428 INFO [RS:1;229bab1f9d30:41705 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,41705,1733691722046/229bab1f9d30%2C41705%2C1733691722046.1733691723411 2024-12-08T21:02:03,428 DEBUG [RS:0;229bab1f9d30:45313 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T21:02:03,431 DEBUG [RS:1;229bab1f9d30:41705 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T21:02:03,593 DEBUG [229bab1f9d30:37617 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-08T21:02:03,605 DEBUG [229bab1f9d30:37617 {}] balancer.BalancerClusterState(204): Hosts are {229bab1f9d30=0} racks are {/default-rack=0} 2024-12-08T21:02:03,608 DEBUG [229bab1f9d30:37617 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T21:02:03,608 DEBUG [229bab1f9d30:37617 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T21:02:03,608 DEBUG [229bab1f9d30:37617 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T21:02:03,608 DEBUG [229bab1f9d30:37617 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T21:02:03,608 INFO [229bab1f9d30:37617 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T21:02:03,608 INFO [229bab1f9d30:37617 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T21:02:03,609 DEBUG [229bab1f9d30:37617 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T21:02:03,614 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=229bab1f9d30,45313,1733691721907 2024-12-08T21:02:03,618 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 229bab1f9d30,45313,1733691721907, state=OPENING 2024-12-08T21:02:03,661 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T21:02:03,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:03,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:03,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:03,672 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:03,672 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:03,672 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:03,675 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T21:02:03,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=229bab1f9d30,45313,1733691721907}] 2024-12-08T21:02:03,856 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T21:02:03,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55437, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T21:02:03,866 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T21:02:03,866 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:03,867 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T21:02:03,868 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=229bab1f9d30%2C45313%2C1733691721907.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,45313,1733691721907, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs, maxLogs=55 2024-12-08T21:02:03,885 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,45313,1733691721907/229bab1f9d30%2C45313%2C1733691721907.meta.1733691723870.meta 2024-12-08T21:02:03,885 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T21:02:03,885 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:03,887 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T21:02:03,889 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T21:02:03,893 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T21:02:03,897 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T21:02:03,897 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:03,897 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T21:02:03,897 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T21:02:03,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T21:02:03,899 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T21:02:03,900 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:03,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:03,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T21:02:03,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T21:02:03,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:03,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:03,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T21:02:03,902 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T21:02:03,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:03,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:03,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T21:02:03,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T21:02:03,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:03,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:03,903 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T21:02:03,904 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740 2024-12-08T21:02:03,905 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740 2024-12-08T21:02:03,905 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T21:02:03,905 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T21:02:03,905 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:02:03,906 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T21:02:03,907 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65530575, jitterRate=-0.023518338799476624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:02:03,907 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T21:02:03,908 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733691723898Writing region info on filesystem at 1733691723898Initializing all the Stores at 1733691723898Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691723898Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691723898Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691723898Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691723898Cleaning up temporary data from old regions at 1733691723905 (+7 ms)Running coprocessor post-open hooks at 1733691723907 (+2 ms)Region opened successfully at 1733691723908 (+1 ms) 2024-12-08T21:02:03,913 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733691723848 2024-12-08T21:02:03,922 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T21:02:03,922 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T21:02:03,923 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,45313,1733691721907 2024-12-08T21:02:03,924 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 229bab1f9d30,45313,1733691721907, state=OPEN 2024-12-08T21:02:03,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:02:03,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:02:03,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:02:03,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:03,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:03,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:03,937 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=229bab1f9d30,45313,1733691721907 2024-12-08T21:02:03,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T21:02:03,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=229bab1f9d30,45313,1733691721907 in 259 msec 2024-12-08T21:02:03,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T21:02:03,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 765 msec 2024-12-08T21:02:03,946 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:03,946 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T21:02:03,965 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:02:03,966 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=229bab1f9d30,45313,1733691721907, seqNum=-1] 2024-12-08T21:02:03,986 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:02:03,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52211, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:02:04,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0060 sec 2024-12-08T21:02:04,004 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733691724004, completionTime=-1 2024-12-08T21:02:04,006 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-08T21:02:04,006 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T21:02:04,029 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-08T21:02:04,029 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733691784029 2024-12-08T21:02:04,029 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733691844029 2024-12-08T21:02:04,029 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-08T21:02:04,031 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-08T21:02:04,037 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,37617,1733691721113-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:04,037 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,37617,1733691721113-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:04,037 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,37617,1733691721113-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:04,038 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-229bab1f9d30:37617, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:04,039 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:04,039 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:04,045 DEBUG [master/229bab1f9d30:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T21:02:04,067 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.896sec 2024-12-08T21:02:04,069 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T21:02:04,070 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T21:02:04,071 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T21:02:04,071 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T21:02:04,071 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T21:02:04,072 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,37617,1733691721113-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:02:04,072 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,37617,1733691721113-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T21:02:04,073 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T21:02:04,074 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T21:02:04,075 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,37617,1733691721113-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:04,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109a9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:02:04,119 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T21:02:04,119 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T21:02:04,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 229bab1f9d30,37617,-1 for getting cluster id 2024-12-08T21:02:04,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:02:04,131 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c36c25c1-e421-4ea6-be64-25b589071f61' 2024-12-08T21:02:04,134 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:02:04,134 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c36c25c1-e421-4ea6-be64-25b589071f61" 2024-12-08T21:02:04,137 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322adfe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:02:04,137 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [229bab1f9d30,37617,-1] 2024-12-08T21:02:04,139 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:02:04,141 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:04,142 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:02:04,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55969d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:02:04,145 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:02:04,150 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=229bab1f9d30,45313,1733691721907, seqNum=-1] 2024-12-08T21:02:04,151 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:02:04,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:02:04,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=229bab1f9d30,37617,1733691721113 2024-12-08T21:02:04,176 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:02:04,179 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 229bab1f9d30,37617,1733691721113 2024-12-08T21:02:04,181 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a9f32cf 2024-12-08T21:02:04,182 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:02:04,185 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:02:04,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733691724172', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:02:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733691724172 2024-12-08T21:02:04,210 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733691724172 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:02:04,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733691724172" procId is: 4 2024-12-08T21:02:04,212 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:04,215 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733691724172 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:02:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:02:04,232 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733691724172-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 76baccb30318bcc074afeb7ea0c79c19, NAME => 'testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733691724172', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 2024-12-08T21:02:04,237 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733691724172-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:04,237 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733691724172-pool-0 {}] regionserver.HRegion(1722): Closing 76baccb30318bcc074afeb7ea0c79c19, disabling compactions & flushes 2024-12-08T21:02:04,237 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733691724172-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:04,237 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733691724172-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:04,238 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733691724172-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. after waiting 1 ms 2024-12-08T21:02:04,238 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733691724172-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:04,238 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733691724172-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:04,238 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733691724172-pool-0 {}] regionserver.HRegion(1676): Region close journal for 76baccb30318bcc074afeb7ea0c79c19: Waiting for close lock at 1733691724237Disabling compacts and flushes for region at 1733691724237Disabling writes for close at 1733691724238 (+1 ms)Writing region close event to WAL at 1733691724238Closed at 1733691724238 2024-12-08T21:02:04,239 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733691724172 execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:02:04,244 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733691724239"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691724239"}]},"ts":"1733691724239"} 2024-12-08T21:02:04,248 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:02:04,249 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733691724172 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:02:04,251 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733691724172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691724249"}]},"ts":"1733691724249"} 2024-12-08T21:02:04,254 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733691724172, state=ENABLING in hbase:meta 2024-12-08T21:02:04,255 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {229bab1f9d30=0} racks are {/default-rack=0} 2024-12-08T21:02:04,255 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T21:02:04,255 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T21:02:04,255 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T21:02:04,255 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T21:02:04,255 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T21:02:04,255 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T21:02:04,256 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T21:02:04,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, ASSIGN}] 2024-12-08T21:02:04,258 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, ASSIGN 2024-12-08T21:02:04,259 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, ASSIGN; state=OFFLINE, location=229bab1f9d30,45313,1733691721907; forceNewPlan=false, retain=false 2024-12-08T21:02:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:02:04,413 INFO [229bab1f9d30:37617 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T21:02:04,415 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=76baccb30318bcc074afeb7ea0c79c19, regionState=OPENING, regionLocation=229bab1f9d30,45313,1733691721907 2024-12-08T21:02:04,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, ASSIGN because future has completed 2024-12-08T21:02:04,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76baccb30318bcc074afeb7ea0c79c19, server=229bab1f9d30,45313,1733691721907}] 2024-12-08T21:02:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:02:04,583 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:04,584 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 76baccb30318bcc074afeb7ea0c79c19, NAME => 'testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:04,585 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733691724172 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,586 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:04,586 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,586 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,588 INFO [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,592 INFO [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76baccb30318bcc074afeb7ea0c79c19 columnFamilyName family 2024-12-08T21:02:04,592 DEBUG [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:04,592 INFO [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] regionserver.HStore(327): Store=76baccb30318bcc074afeb7ea0c79c19/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:04,592 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,593 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,593 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,593 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,593 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,594 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,600 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:02:04,600 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 76baccb30318bcc074afeb7ea0c79c19; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65352751, jitterRate=-0.02616812288761139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:02:04,600 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:04,601 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 76baccb30318bcc074afeb7ea0c79c19: Running coprocessor pre-open hook at 1733691724587Writing region info on filesystem at 1733691724587Initializing all the Stores at 1733691724587Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691724588 (+1 ms)Cleaning up temporary data from old regions at 1733691724593 (+5 ms)Running coprocessor post-open hooks at 1733691724600 (+7 ms)Region opened successfully at 1733691724601 (+1 ms) 2024-12-08T21:02:04,602 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19., pid=6, masterSystemTime=1733691724576 2024-12-08T21:02:04,605 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:04,605 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:04,606 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=76baccb30318bcc074afeb7ea0c79c19, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,45313,1733691721907 2024-12-08T21:02:04,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76baccb30318bcc074afeb7ea0c79c19, server=229bab1f9d30,45313,1733691721907 because future has completed 2024-12-08T21:02:04,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T21:02:04,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 76baccb30318bcc074afeb7ea0c79c19, server=229bab1f9d30,45313,1733691721907 in 188 msec 2024-12-08T21:02:04,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T21:02:04,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, ASSIGN in 355 msec 2024-12-08T21:02:04,615 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733691724172 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:02:04,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733691724172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691724615"}]},"ts":"1733691724615"} 2024-12-08T21:02:04,617 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733691724172, state=ENABLED in hbase:meta 2024-12-08T21:02:04,619 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733691724172 execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:02:04,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733691724172 in 415 msec 2024-12-08T21:02:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:02:04,851 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733691724172 get assigned. Timeout = 60000ms 2024-12-08T21:02:04,851 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733691724172 completed 2024-12-08T21:02:04,857 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:02:04,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733691724172 assigned to meta. Checking AM states. 2024-12-08T21:02:04,866 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:02:04,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733691724172 assigned. 2024-12-08T21:02:04,874 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733691724172', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19., hostname=229bab1f9d30,45313,1733691721907, seqNum=2] 2024-12-08T21:02:04,885 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 76baccb30318bcc074afeb7ea0c79c19 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-08T21:02:04,906 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 486ea215c6ca4771a1da83c94a05ca53 2024-12-08T21:02:04,937 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/.tmp/family/486ea215c6ca4771a1da83c94a05ca53 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-08T21:02:04,941 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/.tmp/family/486ea215c6ca4771a1da83c94a05ca53 2024-12-08T21:02:04,970 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/.tmp/family/486ea215c6ca4771a1da83c94a05ca53 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53 2024-12-08T21:02:04,971 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53, delay=952 ms 2024-12-08T21:02:04,972 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53, entries=2, sequenceid=6, filesize=4.9 K 2024-12-08T21:02:04,977 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 76baccb30318bcc074afeb7ea0c79c19 in 93ms, sequenceid=6, compaction requested=false 2024-12-08T21:02:04,978 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733691724172' 2024-12-08T21:02:04,980 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 76baccb30318bcc074afeb7ea0c79c19: 2024-12-08T21:02:05,928 DEBUG [hfile-prefetch-1733691724971 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 486ea215c6ca4771a1da83c94a05ca53_0. 2024-12-08T21:02:05,935 DEBUG [hfile-prefetch-1733691724971 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 486ea215c6ca4771a1da83c94a05ca53_117. 2024-12-08T21:02:05,937 DEBUG [hfile-prefetch-1733691724971 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53, with total blocks 2, and data blocks 1 2024-12-08T21:02:05,937 DEBUG [hfile-prefetch-1733691724971 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-08T21:02:05,938 DEBUG [hfile-prefetch-1733691724971 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-08T21:02:05,939 DEBUG [hfile-prefetch-1733691724971 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53. Will sleep 100ms and try the verification again. 2024-12-08T21:02:05,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=76baccb30318bcc074afeb7ea0c79c19, source=229bab1f9d30,45313,1733691721907, destination=229bab1f9d30,41705,1733691722046, warming up region on 229bab1f9d30,41705,1733691722046 2024-12-08T21:02:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T21:02:05,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=76baccb30318bcc074afeb7ea0c79c19, source=229bab1f9d30,45313,1733691721907, destination=229bab1f9d30,41705,1733691722046, running balancer 2024-12-08T21:02:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, REOPEN/MOVE 2024-12-08T21:02:05,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55671, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T21:02:05,992 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, REOPEN/MOVE 2024-12-08T21:02:05,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.HRegion(7855): Warmup {ENCODED => 76baccb30318bcc074afeb7ea0c79c19, NAME => 'testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:05,994 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=76baccb30318bcc074afeb7ea0c79c19, regionState=CLOSING, regionLocation=229bab1f9d30,45313,1733691721907 2024-12-08T21:02:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:05,995 INFO [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:05,995 INFO [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76baccb30318bcc074afeb7ea0c79c19 columnFamilyName family 2024-12-08T21:02:05,996 DEBUG [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:05,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, REOPEN/MOVE because future has completed 2024-12-08T21:02:05,997 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-08T21:02:05,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76baccb30318bcc074afeb7ea0c79c19, server=229bab1f9d30,45313,1733691721907}] 2024-12-08T21:02:05,999 DEBUG [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53 2024-12-08T21:02:05,999 INFO [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] regionserver.HStore(327): Store=76baccb30318bcc074afeb7ea0c79c19/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.HRegion(1722): Closing 76baccb30318bcc074afeb7ea0c79c19, disabling compactions & flushes 2024-12-08T21:02:05,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. after waiting 0 ms 2024-12-08T21:02:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,001 DEBUG [StoreFileCloser-76baccb30318bcc074afeb7ea0c79c19-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53 2024-12-08T21:02:06,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] regionserver.HRegion(1676): Region close journal for 76baccb30318bcc074afeb7ea0c79c19: Waiting for close lock at 1733691725999Disabling compacts and flushes for region at 1733691725999Disabling writes for close at 1733691725999Writing region close event to WAL at 1733691726001 (+2 ms)Closed at 1733691726001 2024-12-08T21:02:06,002 WARN [hfile-prefetch-1733691724971 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-08T21:02:06,003 DEBUG [hfile-prefetch-1733691724971 {}] hfile.PrefetchExecutor(121): Prefetch completed for 486ea215c6ca4771a1da83c94a05ca53 2024-12-08T21:02:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37617 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-08T21:02:06,099 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-08T21:02:06,099 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-08T21:02:06,100 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T21:02:06,100 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T21:02:06,112 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-08T21:02:06,112 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-08T21:02:06,113 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T21:02:06,113 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T21:02:06,120 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T21:02:06,120 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733691726099 2024-12-08T21:02:06,121 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-08T21:02:06,123 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-08T21:02:06,123 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733691726112 2024-12-08T21:02:06,123 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-08T21:02:06,156 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,156 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-08T21:02:06,157 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 76baccb30318bcc074afeb7ea0c79c19, disabling compactions & flushes 2024-12-08T21:02:06,157 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,157 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,157 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. after waiting 0 ms 2024-12-08T21:02:06,158 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,160 DEBUG [StoreFileCloser-76baccb30318bcc074afeb7ea0c79c19-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 486ea215c6ca4771a1da83c94a05ca53, starting offset: 0, end offset: 9223372036854775807 2024-12-08T21:02:06,161 DEBUG [StoreFileCloser-76baccb30318bcc074afeb7ea0c79c19-family-1 {}] bucket.BucketCache(856): removed key 486ea215c6ca4771a1da83c94a05ca53_0 from back map with offset lock 2094080 in the evict process 2024-12-08T21:02:06,169 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-08T21:02:06,171 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,171 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 76baccb30318bcc074afeb7ea0c79c19: Waiting for close lock at 1733691726157Running coprocessor pre-close hooks at 1733691726157Disabling compacts and flushes for region at 1733691726157Disabling writes for close at 1733691726157Writing region close event to WAL at 1733691726162 (+5 ms)Running coprocessor post-close hooks at 1733691726169 (+7 ms)Closed at 1733691726171 (+2 ms) 2024-12-08T21:02:06,171 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 76baccb30318bcc074afeb7ea0c79c19 move to 229bab1f9d30,41705,1733691722046 record at close sequenceid=6 2024-12-08T21:02:06,175 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,176 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=76baccb30318bcc074afeb7ea0c79c19, regionState=CLOSED 2024-12-08T21:02:06,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76baccb30318bcc074afeb7ea0c79c19, server=229bab1f9d30,45313,1733691721907 because future has completed 2024-12-08T21:02:06,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T21:02:06,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 76baccb30318bcc074afeb7ea0c79c19, server=229bab1f9d30,45313,1733691721907 in 182 msec 2024-12-08T21:02:06,184 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, REOPEN/MOVE; state=CLOSED, location=229bab1f9d30,41705,1733691722046; forceNewPlan=false, retain=false 2024-12-08T21:02:06,334 INFO [229bab1f9d30:37617 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T21:02:06,335 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=76baccb30318bcc074afeb7ea0c79c19, regionState=OPENING, regionLocation=229bab1f9d30,41705,1733691722046 2024-12-08T21:02:06,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, REOPEN/MOVE because future has completed 2024-12-08T21:02:06,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76baccb30318bcc074afeb7ea0c79c19, server=229bab1f9d30,41705,1733691722046}] 2024-12-08T21:02:06,496 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,497 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 76baccb30318bcc074afeb7ea0c79c19, NAME => 'testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:06,497 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733691724172 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,497 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:06,498 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,498 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,499 INFO [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,501 INFO [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76baccb30318bcc074afeb7ea0c79c19 columnFamilyName family 2024-12-08T21:02:06,501 DEBUG [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:06,504 DEBUG [StoreFileOpener-76baccb30318bcc074afeb7ea0c79c19-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53, delay=909 ms 2024-12-08T21:02:06,506 DEBUG [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53 2024-12-08T21:02:06,506 INFO [StoreOpener-76baccb30318bcc074afeb7ea0c79c19-1 {}] regionserver.HStore(327): Store=76baccb30318bcc074afeb7ea0c79c19/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:06,507 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,508 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,508 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,509 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,509 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,514 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,515 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 76baccb30318bcc074afeb7ea0c79c19; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69095622, jitterRate=0.029605001211166382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:02:06,515 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,516 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 76baccb30318bcc074afeb7ea0c79c19: Running coprocessor pre-open hook at 1733691726498Writing region info on filesystem at 1733691726498Initializing all the Stores at 1733691726499 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691726499Cleaning up temporary data from old regions at 1733691726509 (+10 ms)Running coprocessor post-open hooks at 1733691726515 (+6 ms)Region opened successfully at 1733691726516 (+1 ms) 2024-12-08T21:02:06,517 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19., pid=9, masterSystemTime=1733691726492 2024-12-08T21:02:06,519 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,519 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=76baccb30318bcc074afeb7ea0c79c19, regionState=OPEN, openSeqNum=10, regionLocation=229bab1f9d30,41705,1733691722046 2024-12-08T21:02:06,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76baccb30318bcc074afeb7ea0c79c19, server=229bab1f9d30,41705,1733691722046 because future has completed 2024-12-08T21:02:06,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-08T21:02:06,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 76baccb30318bcc074afeb7ea0c79c19, server=229bab1f9d30,41705,1733691722046 in 185 msec 2024-12-08T21:02:06,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733691724172, region=76baccb30318bcc074afeb7ea0c79c19, REOPEN/MOVE in 535 msec 2024-12-08T21:02:06,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T21:02:06,546 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:02:06,546 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:02:06,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:06,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:06,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:02:06,553 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T21:02:06,553 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-12-08T21:02:06,553 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=229bab1f9d30,37617,1733691721113 2024-12-08T21:02:06,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:06,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:06,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:06,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:06,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:06,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:06,586 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T21:02:06,587 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:02:06,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:06,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:06,587 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:02:06,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:06,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:06,588 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '229bab1f9d30,45313,1733691721907' ***** 2024-12-08T21:02:06,589 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T21:02:06,589 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '229bab1f9d30,41705,1733691722046' ***** 2024-12-08T21:02:06,589 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T21:02:06,589 INFO [RS:1;229bab1f9d30:41705 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T21:02:06,589 INFO [RS:0;229bab1f9d30:45313 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T21:02:06,590 INFO [RS:1;229bab1f9d30:41705 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.persistence 2024-12-08T21:02:06,590 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T21:02:06,590 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T21:02:06,590 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T21:02:06,590 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T21:02:06,590 INFO [RS:0;229bab1f9d30:45313 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/bucket.persistence 2024-12-08T21:02:06,590 INFO [RS:0;229bab1f9d30:45313 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T21:02:06,590 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T21:02:06,591 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T21:02:06,591 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T21:02:06,591 INFO [RS:1;229bab1f9d30:41705 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T21:02:06,591 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T21:02:06,600 DEBUG [RS:0;229bab1f9d30:45313 {}] bucket.BucketCache(1487): Thread RS:0;229bab1f9d30:45313 started persisting bucket cache to file 2024-12-08T21:02:06,601 DEBUG [RS:0;229bab1f9d30:45313 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T21:02:06,601 DEBUG [RS:0;229bab1f9d30:45313 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T21:02:06,601 DEBUG [RS:1;229bab1f9d30:41705 {}] bucket.BucketCache(1487): Thread RS:1;229bab1f9d30:41705 started persisting bucket cache to file 2024-12-08T21:02:06,601 DEBUG [RS:1;229bab1f9d30:41705 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T21:02:06,601 DEBUG [RS:1;229bab1f9d30:41705 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T21:02:06,604 DEBUG [RS:0;229bab1f9d30:45313 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T21:02:06,604 DEBUG [RS:0;229bab1f9d30:45313 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733691726600 2024-12-08T21:02:06,603 ERROR [RS:1;229bab1f9d30:41705 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:02:06,604 DEBUG [RS:0;229bab1f9d30:45313 {}] bucket.BucketCache(1509): Thread RS:0;229bab1f9d30:45313 finished persisting bucket cache to file, renaming 2024-12-08T21:02:06,604 DEBUG [RS:1;229bab1f9d30:41705 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T21:02:06,604 DEBUG [RS:1;229bab1f9d30:41705 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733691726601 2024-12-08T21:02:06,604 DEBUG [RS:1;229bab1f9d30:41705 {}] bucket.BucketCache(1509): Thread RS:1;229bab1f9d30:41705 finished persisting bucket cache to file, renaming 2024-12-08T21:02:06,604 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T21:02:06,604 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T21:02:06,604 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T21:02:06,604 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T21:02:06,604 INFO [RS:0;229bab1f9d30:45313 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T21:02:06,604 INFO [RS:1;229bab1f9d30:41705 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T21:02:06,605 INFO [RS:0;229bab1f9d30:45313 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T21:02:06,605 INFO [RS:1;229bab1f9d30:41705 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T21:02:06,605 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(959): stopping server 229bab1f9d30,45313,1733691721907 2024-12-08T21:02:06,605 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(3091): Received CLOSE for 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,605 INFO [RS:0;229bab1f9d30:45313 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:02:06,605 INFO [RS:0;229bab1f9d30:45313 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;229bab1f9d30:45313. 2024-12-08T21:02:06,605 DEBUG [RS:0;229bab1f9d30:45313 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:02:06,605 DEBUG [RS:0;229bab1f9d30:45313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:06,605 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T21:02:06,605 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(959): stopping server 229bab1f9d30,41705,1733691722046 2024-12-08T21:02:06,605 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T21:02:06,605 INFO [RS:1;229bab1f9d30:41705 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:02:06,605 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T21:02:06,605 INFO [RS:1;229bab1f9d30:41705 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;229bab1f9d30:41705. 2024-12-08T21:02:06,606 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T21:02:06,606 DEBUG [RS:1;229bab1f9d30:41705 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:02:06,606 DEBUG [RS:1;229bab1f9d30:41705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:06,606 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T21:02:06,606 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T21:02:06,606 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T21:02:06,606 DEBUG [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T21:02:06,606 DEBUG [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(1325): Online Regions={76baccb30318bcc074afeb7ea0c79c19=testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19.} 2024-12-08T21:02:06,606 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T21:02:06,606 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 76baccb30318bcc074afeb7ea0c79c19, disabling compactions & flushes 2024-12-08T21:02:06,606 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,606 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,606 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T21:02:06,606 DEBUG [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:02:06,606 DEBUG [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(1351): Waiting on 76baccb30318bcc074afeb7ea0c79c19 2024-12-08T21:02:06,606 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. after waiting 0 ms 2024-12-08T21:02:06,606 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T21:02:06,606 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,606 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T21:02:06,606 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T21:02:06,606 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T21:02:06,607 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-08T21:02:06,607 DEBUG [StoreFileCloser-76baccb30318bcc074afeb7ea0c79c19-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/family/486ea215c6ca4771a1da83c94a05ca53 2024-12-08T21:02:06,612 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnRegionMove1733691724172/76baccb30318bcc074afeb7ea0c79c19/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-08T21:02:06,612 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,612 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 76baccb30318bcc074afeb7ea0c79c19: Waiting for close lock at 1733691726605Running coprocessor pre-close hooks at 1733691726606 (+1 ms)Disabling compacts and flushes for region at 1733691726606Disabling writes for close at 1733691726606Writing region close event to WAL at 1733691726607 (+1 ms)Running coprocessor post-close hooks at 1733691726612 (+5 ms)Closed at 1733691726612 2024-12-08T21:02:06,613 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19. 2024-12-08T21:02:06,616 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 40b506c1de9540e093a5031308e57b93 2024-12-08T21:02:06,630 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/info/40b506c1de9540e093a5031308e57b93 is 205, key is testBlockEvictionOnRegionMove1733691724172,,1733691724186.76baccb30318bcc074afeb7ea0c79c19./info:regioninfo/1733691726519/Put/seqid=0 2024-12-08T21:02:06,631 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/info/40b506c1de9540e093a5031308e57b93 2024-12-08T21:02:06,638 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 26e713254a9747b79a0cfccbc844a2b3 2024-12-08T21:02:06,649 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/ns/26e713254a9747b79a0cfccbc844a2b3 is 43, key is default/ns:d/1733691723992/Put/seqid=0 2024-12-08T21:02:06,650 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/ns/26e713254a9747b79a0cfccbc844a2b3 2024-12-08T21:02:06,657 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c30a7e49d0bf43f58ee45c13d311bb68 2024-12-08T21:02:06,660 INFO [regionserver/229bab1f9d30:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:06,661 INFO [regionserver/229bab1f9d30:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:06,670 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/table/c30a7e49d0bf43f58ee45c13d311bb68 is 78, key is testBlockEvictionOnRegionMove1733691724172/table:state/1733691724615/Put/seqid=0 2024-12-08T21:02:06,671 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/table/c30a7e49d0bf43f58ee45c13d311bb68 2024-12-08T21:02:06,673 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/info/40b506c1de9540e093a5031308e57b93 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/info/40b506c1de9540e093a5031308e57b93 2024-12-08T21:02:06,674 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/info/40b506c1de9540e093a5031308e57b93, delay=939 ms 2024-12-08T21:02:06,675 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/info/40b506c1de9540e093a5031308e57b93, entries=15, sequenceid=15, filesize=7.6 K 2024-12-08T21:02:06,676 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/ns/26e713254a9747b79a0cfccbc844a2b3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/ns/26e713254a9747b79a0cfccbc844a2b3 2024-12-08T21:02:06,677 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/ns/26e713254a9747b79a0cfccbc844a2b3, delay=955 ms 2024-12-08T21:02:06,678 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/ns/26e713254a9747b79a0cfccbc844a2b3, entries=2, sequenceid=15, filesize=5.0 K 2024-12-08T21:02:06,678 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/table/c30a7e49d0bf43f58ee45c13d311bb68 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/table/c30a7e49d0bf43f58ee45c13d311bb68 2024-12-08T21:02:06,680 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/table/c30a7e49d0bf43f58ee45c13d311bb68, delay=936 ms 2024-12-08T21:02:06,681 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/table/c30a7e49d0bf43f58ee45c13d311bb68, entries=2, sequenceid=15, filesize=5.3 K 2024-12-08T21:02:06,681 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=15, compaction requested=false 2024-12-08T21:02:06,681 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T21:02:06,683 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/info/40b506c1de9540e093a5031308e57b93 2024-12-08T21:02:06,684 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/ns/26e713254a9747b79a0cfccbc844a2b3 2024-12-08T21:02:06,686 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/table/c30a7e49d0bf43f58ee45c13d311bb68 2024-12-08T21:02:06,690 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-08T21:02:06,690 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T21:02:06,690 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T21:02:06,690 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733691726606Running coprocessor pre-close hooks at 1733691726606Disabling compacts and flushes for region at 1733691726606Disabling writes for close at 1733691726606Obtaining lock to block concurrent updates at 1733691726607 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733691726607Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733691726607Flushing stores of hbase:meta,,1.1588230740 at 1733691726608 (+1 ms)Flushing 1588230740/info: creating writer at 1733691726608Flushing 1588230740/info: appending metadata at 1733691726629 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733691726629Flushing 1588230740/ns: creating writer at 1733691726633 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733691726648 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733691726648Flushing 1588230740/table: creating writer at 1733691726652 (+4 ms)Flushing 1588230740/table: appending metadata at 1733691726670 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733691726670Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@451277f2: reopening flushed file at 1733691726673 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aec51c0: reopening flushed file at 1733691726676 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c723c46: reopening flushed file at 1733691726678 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=15, compaction requested=false at 1733691726681 (+3 ms)Writing region close event to WAL at 1733691726686 (+5 ms)Running coprocessor post-close hooks at 1733691726690 (+4 ms)Closed at 1733691726690 2024-12-08T21:02:06,691 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T21:02:06,807 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(976): stopping server 229bab1f9d30,41705,1733691722046; all regions closed. 2024-12-08T21:02:06,807 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(976): stopping server 229bab1f9d30,45313,1733691721907; all regions closed. 2024-12-08T21:02:06,815 DEBUG [RS:1;229bab1f9d30:41705 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs 2024-12-08T21:02:06,815 DEBUG [RS:0;229bab1f9d30:45313 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs 2024-12-08T21:02:06,816 INFO [RS:1;229bab1f9d30:41705 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 229bab1f9d30%2C41705%2C1733691722046:(num 1733691723411) 2024-12-08T21:02:06,816 INFO [RS:0;229bab1f9d30:45313 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 229bab1f9d30%2C45313%2C1733691721907.meta:.meta(num 1733691723870) 2024-12-08T21:02:06,816 DEBUG [RS:1;229bab1f9d30:41705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:06,816 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:06,816 INFO [RS:1;229bab1f9d30:41705 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:02:06,817 INFO [RS:1;229bab1f9d30:41705 {}] hbase.ChoreService(370): Chore service for: regionserver/229bab1f9d30:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T21:02:06,817 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T21:02:06,817 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T21:02:06,817 INFO [regionserver/229bab1f9d30:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:02:06,817 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T21:02:06,817 INFO [RS:1;229bab1f9d30:41705 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:02:06,818 INFO [RS:1;229bab1f9d30:41705 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41705 2024-12-08T21:02:06,818 DEBUG [RS:0;229bab1f9d30:45313 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs 2024-12-08T21:02:06,819 INFO [RS:0;229bab1f9d30:45313 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 229bab1f9d30%2C45313%2C1733691721907:(num 1733691723411) 2024-12-08T21:02:06,819 DEBUG [RS:0;229bab1f9d30:45313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:06,819 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:06,819 INFO [RS:0;229bab1f9d30:45313 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:02:06,819 INFO [RS:0;229bab1f9d30:45313 {}] hbase.ChoreService(370): Chore service for: regionserver/229bab1f9d30:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T21:02:06,819 INFO [RS:0;229bab1f9d30:45313 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:02:06,819 INFO [regionserver/229bab1f9d30:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:02:06,820 INFO [RS:0;229bab1f9d30:45313 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45313 2024-12-08T21:02:06,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:02:06,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/229bab1f9d30,41705,1733691722046 2024-12-08T21:02:06,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/229bab1f9d30,45313,1733691721907 2024-12-08T21:02:06,835 INFO [RS:1;229bab1f9d30:41705 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:02:06,835 INFO [RS:0;229bab1f9d30:45313 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:02:06,844 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [229bab1f9d30,45313,1733691721907] 2024-12-08T21:02:06,852 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/229bab1f9d30,45313,1733691721907 already deleted, retry=false 2024-12-08T21:02:06,852 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 229bab1f9d30,45313,1733691721907 expired; onlineServers=1 2024-12-08T21:02:06,852 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [229bab1f9d30,41705,1733691722046] 2024-12-08T21:02:06,860 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/229bab1f9d30,41705,1733691722046 already deleted, retry=false 2024-12-08T21:02:06,860 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 229bab1f9d30,41705,1733691722046 expired; onlineServers=0 2024-12-08T21:02:06,860 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '229bab1f9d30,37617,1733691721113' ***** 2024-12-08T21:02:06,860 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T21:02:06,861 INFO [M:0;229bab1f9d30:37617 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:02:06,861 INFO [M:0;229bab1f9d30:37617 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:02:06,861 DEBUG [M:0;229bab1f9d30:37617 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T21:02:06,861 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T21:02:06,861 DEBUG [M:0;229bab1f9d30:37617 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T21:02:06,861 DEBUG [master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.small.0-1733691723099 {}] cleaner.HFileCleaner(306): Exit Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.small.0-1733691723099,5,FailOnTimeoutGroup] 2024-12-08T21:02:06,861 DEBUG [master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.large.0-1733691723093 {}] cleaner.HFileCleaner(306): Exit Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.large.0-1733691723093,5,FailOnTimeoutGroup] 2024-12-08T21:02:06,861 INFO [M:0;229bab1f9d30:37617 {}] hbase.ChoreService(370): Chore service for: master/229bab1f9d30:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T21:02:06,862 INFO [M:0;229bab1f9d30:37617 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:02:06,862 DEBUG [M:0;229bab1f9d30:37617 {}] master.HMaster(1795): Stopping service threads 2024-12-08T21:02:06,862 INFO [M:0;229bab1f9d30:37617 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T21:02:06,862 INFO [M:0;229bab1f9d30:37617 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T21:02:06,862 INFO [M:0;229bab1f9d30:37617 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T21:02:06,862 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T21:02:06,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T21:02:06,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:06,869 DEBUG [M:0;229bab1f9d30:37617 {}] zookeeper.ZKUtil(347): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T21:02:06,869 WARN [M:0;229bab1f9d30:37617 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T21:02:06,870 INFO [M:0;229bab1f9d30:37617 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/.lastflushedseqids 2024-12-08T21:02:06,876 INFO [M:0;229bab1f9d30:37617 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T21:02:06,877 INFO [M:0;229bab1f9d30:37617 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T21:02:06,877 DEBUG [M:0;229bab1f9d30:37617 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T21:02:06,877 INFO [M:0;229bab1f9d30:37617 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:06,877 DEBUG [M:0;229bab1f9d30:37617 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:06,877 DEBUG [M:0;229bab1f9d30:37617 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T21:02:06,877 DEBUG [M:0;229bab1f9d30:37617 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:06,877 INFO [M:0;229bab1f9d30:37617 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-08T21:02:06,885 DEBUG [M:0;229bab1f9d30:37617 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6586ca83d0f8434681e519e6756b3b3a 2024-12-08T21:02:06,897 DEBUG [M:0;229bab1f9d30:37617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6586ca83d0f8434681e519e6756b3b3a is 82, key is hbase:meta,,1/info:regioninfo/1733691723922/Put/seqid=0 2024-12-08T21:02:06,898 INFO [M:0;229bab1f9d30:37617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6586ca83d0f8434681e519e6756b3b3a 2024-12-08T21:02:06,905 DEBUG [M:0;229bab1f9d30:37617 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7afbee1edbf049378f511418c62cb6b1 2024-12-08T21:02:06,916 DEBUG [M:0;229bab1f9d30:37617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7afbee1edbf049378f511418c62cb6b1 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733691724620/Put/seqid=0 2024-12-08T21:02:06,916 INFO [M:0;229bab1f9d30:37617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7afbee1edbf049378f511418c62cb6b1 2024-12-08T21:02:06,923 DEBUG [M:0;229bab1f9d30:37617 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3eb92fed32bb408f99f37001282bd879 2024-12-08T21:02:06,933 DEBUG [M:0;229bab1f9d30:37617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3eb92fed32bb408f99f37001282bd879 is 69, key is 229bab1f9d30,41705,1733691722046/rs:state/1733691723155/Put/seqid=0 2024-12-08T21:02:06,934 INFO [M:0;229bab1f9d30:37617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3eb92fed32bb408f99f37001282bd879 2024-12-08T21:02:06,936 DEBUG [M:0;229bab1f9d30:37617 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6586ca83d0f8434681e519e6756b3b3a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6586ca83d0f8434681e519e6756b3b3a 2024-12-08T21:02:06,937 INFO [M:0;229bab1f9d30:37617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6586ca83d0f8434681e519e6756b3b3a, entries=8, sequenceid=84, filesize=5.5 K 2024-12-08T21:02:06,938 DEBUG [M:0;229bab1f9d30:37617 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7afbee1edbf049378f511418c62cb6b1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7afbee1edbf049378f511418c62cb6b1 2024-12-08T21:02:06,939 INFO [M:0;229bab1f9d30:37617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7afbee1edbf049378f511418c62cb6b1, entries=9, sequenceid=84, filesize=6.5 K 2024-12-08T21:02:06,939 DEBUG [M:0;229bab1f9d30:37617 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3eb92fed32bb408f99f37001282bd879 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3eb92fed32bb408f99f37001282bd879 2024-12-08T21:02:06,941 INFO [M:0;229bab1f9d30:37617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3eb92fed32bb408f99f37001282bd879, entries=2, sequenceid=84, filesize=5.1 K 2024-12-08T21:02:06,941 INFO [M:0;229bab1f9d30:37617 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 64ms, sequenceid=84, compaction requested=false 2024-12-08T21:02:06,943 INFO [M:0;229bab1f9d30:37617 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:06,944 DEBUG [M:0;229bab1f9d30:37617 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733691726877Disabling compacts and flushes for region at 1733691726877Disabling writes for close at 1733691726877Obtaining lock to block concurrent updates at 1733691726877Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733691726877Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733691726878 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733691726878Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733691726878Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733691726897 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733691726897Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733691726900 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733691726915 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733691726915Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733691726918 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733691726933 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733691726933Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7955e4fb: reopening flushed file at 1733691726935 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c030262: reopening flushed file at 1733691726937 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c5d24a5: reopening flushed file at 1733691726939 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 64ms, sequenceid=84, compaction requested=false at 1733691726942 (+3 ms)Writing region close event to WAL at 1733691726943 (+1 ms)Closed at 1733691726943 2024-12-08T21:02:06,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:06,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41705-0x100074a03310002, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:06,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:06,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100074a03310001, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:06,945 INFO [RS:1;229bab1f9d30:41705 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:02:06,945 INFO [RS:0;229bab1f9d30:45313 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:02:06,945 INFO [RS:1;229bab1f9d30:41705 {}] regionserver.HRegionServer(1031): Exiting; stopping=229bab1f9d30,41705,1733691722046; zookeeper connection closed. 2024-12-08T21:02:06,945 INFO [RS:0;229bab1f9d30:45313 {}] regionserver.HRegionServer(1031): Exiting; stopping=229bab1f9d30,45313,1733691721907; zookeeper connection closed. 2024-12-08T21:02:06,945 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@144e6f53 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@144e6f53 2024-12-08T21:02:06,945 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@337b93f4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@337b93f4 2024-12-08T21:02:06,945 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:02:06,945 INFO [M:0;229bab1f9d30:37617 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T21:02:06,946 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-08T21:02:06,946 INFO [M:0;229bab1f9d30:37617 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37617 2024-12-08T21:02:06,946 INFO [M:0;229bab1f9d30:37617 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:02:07,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:07,086 INFO [M:0;229bab1f9d30:37617 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:02:07,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37617-0x100074a03310000, quorum=127.0.0.1:52417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:07,095 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T21:02:07,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-08T21:02:07,113 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:07,120 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: master/229bab1f9d30:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1733691726504 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/229bab1f9d30:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/229bab1f9d30:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733691726674 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733691724971 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/229bab1f9d30:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1733691726677 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 407), ProcessCount=11 (was 11), AvailableMemoryMB=16869 (was 17253) 2024-12-08T21:02:07,125 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=16869 2024-12-08T21:02:07,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/hadoop-log-dir so I do NOT create it in target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e 2024-12-08T21:02:07,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/hadoop-tmp-dir so I do NOT create it in target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e 2024-12-08T21:02:07,125 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:07,128 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/cluster_33308ff2-d7b8-7535-f71c-fda1bcc9a3c5, deleteOnExit=true 2024-12-08T21:02:07,130 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/cluster_33308ff2-d7b8-7535-f71c-fda1bcc9a3c5/zookeeper_0, clientPort=60212, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/cluster_33308ff2-d7b8-7535-f71c-fda1bcc9a3c5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/cluster_33308ff2-d7b8-7535-f71c-fda1bcc9a3c5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T21:02:07,132 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60212 2024-12-08T21:02:07,143 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 with version=8 2024-12-08T21:02:07,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/hbase-staging 2024-12-08T21:02:07,146 INFO [Time-limited test {}] client.ConnectionUtils(128): master/229bab1f9d30:0 server-side Connection retries=45 2024-12-08T21:02:07,146 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:07,146 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:07,146 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:02:07,146 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:07,146 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:02:07,146 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T21:02:07,146 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:02:07,147 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45245 2024-12-08T21:02:07,148 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45245 connecting to ZooKeeper ensemble=127.0.0.1:60212 2024-12-08T21:02:07,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:452450x0, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:02:07,189 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45245-0x100074a1bb60000 connected 2024-12-08T21:02:07,271 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:07,272 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:07,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:07,275 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200, hbase.cluster.distributed=false 2024-12-08T21:02:07,279 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:02:07,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45245 2024-12-08T21:02:07,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45245 2024-12-08T21:02:07,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45245 2024-12-08T21:02:07,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45245 2024-12-08T21:02:07,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45245 2024-12-08T21:02:07,295 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/229bab1f9d30:0 server-side Connection retries=45 2024-12-08T21:02:07,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:07,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:07,295 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:02:07,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:07,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:02:07,295 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T21:02:07,295 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:02:07,296 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33091 2024-12-08T21:02:07,298 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33091 connecting to ZooKeeper ensemble=127.0.0.1:60212 2024-12-08T21:02:07,298 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:07,299 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:07,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330910x0, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:02:07,305 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:330910x0, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:07,305 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33091-0x100074a1bb60001 connected 2024-12-08T21:02:07,305 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T21:02:07,306 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.cache 2024-12-08T21:02:07,306 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T21:02:07,306 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T21:02:07,306 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T21:02:07,306 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T21:02:07,306 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-08T21:02:07,306 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T21:02:07,307 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T21:02:07,307 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T21:02:07,307 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.persistence is complete. 2024-12-08T21:02:07,307 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T21:02:07,308 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:02:07,309 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33091 2024-12-08T21:02:07,310 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33091 2024-12-08T21:02:07,310 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33091 2024-12-08T21:02:07,310 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33091 2024-12-08T21:02:07,310 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33091 2024-12-08T21:02:07,326 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/229bab1f9d30:0 server-side Connection retries=45 2024-12-08T21:02:07,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:07,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:07,326 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:02:07,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:07,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:02:07,326 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T21:02:07,327 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:02:07,327 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39131 2024-12-08T21:02:07,328 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39131 connecting to ZooKeeper ensemble=127.0.0.1:60212 2024-12-08T21:02:07,329 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:07,329 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:07,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391310x0, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:02:07,338 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39131-0x100074a1bb60002 connected 2024-12-08T21:02:07,338 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:07,338 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T21:02:07,339 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.cache 2024-12-08T21:02:07,339 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T21:02:07,339 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T21:02:07,339 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T21:02:07,340 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T21:02:07,340 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-08T21:02:07,340 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T21:02:07,340 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T21:02:07,340 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.persistence is complete. 2024-12-08T21:02:07,340 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T21:02:07,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T21:02:07,342 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:02:07,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39131 2024-12-08T21:02:07,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39131 2024-12-08T21:02:07,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39131 2024-12-08T21:02:07,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39131 2024-12-08T21:02:07,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39131 2024-12-08T21:02:07,360 DEBUG [M:0;229bab1f9d30:45245 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;229bab1f9d30:45245 2024-12-08T21:02:07,363 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/229bab1f9d30,45245,1733691727145 2024-12-08T21:02:07,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:07,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:07,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:07,372 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/229bab1f9d30,45245,1733691727145 2024-12-08T21:02:07,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T21:02:07,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T21:02:07,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,380 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T21:02:07,381 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/229bab1f9d30,45245,1733691727145 from backup master directory 2024-12-08T21:02:07,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/229bab1f9d30,45245,1733691727145 2024-12-08T21:02:07,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:07,388 WARN [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:02:07,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:07,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:07,388 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=229bab1f9d30,45245,1733691727145 2024-12-08T21:02:07,389 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/hbase.id] with ID: 22cf2d26-7bb4-46b2-9c0c-5322bf0f54c7 2024-12-08T21:02:07,389 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/.tmp/hbase.id 2024-12-08T21:02:07,392 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/hbase.id] 2024-12-08T21:02:07,421 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:07,421 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T21:02:07,423 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-08T21:02:07,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,433 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:02:07,435 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T21:02:07,435 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:07,454 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store 2024-12-08T21:02:07,460 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:07,460 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T21:02:07,460 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:07,460 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:07,461 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T21:02:07,461 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:07,461 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:07,461 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733691727460Disabling compacts and flushes for region at 1733691727460Disabling writes for close at 1733691727461 (+1 ms)Writing region close event to WAL at 1733691727461Closed at 1733691727461 2024-12-08T21:02:07,461 WARN [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/.initializing 2024-12-08T21:02:07,461 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/WALs/229bab1f9d30,45245,1733691727145 2024-12-08T21:02:07,466 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:02:07,470 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=229bab1f9d30%2C45245%2C1733691727145, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/WALs/229bab1f9d30,45245,1733691727145, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/oldWALs, maxLogs=10 2024-12-08T21:02:07,485 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/WALs/229bab1f9d30,45245,1733691727145/229bab1f9d30%2C45245%2C1733691727145.1733691727470 2024-12-08T21:02:07,485 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T21:02:07,486 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:07,486 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:07,486 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,486 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,487 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T21:02:07,491 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:07,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:07,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T21:02:07,495 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:07,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:07,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T21:02:07,499 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:07,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:07,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,502 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T21:02:07,502 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:07,502 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:07,503 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,503 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,503 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,503 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,503 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,504 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:02:07,504 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:07,509 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:02:07,510 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68053548, jitterRate=0.01407688856124878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:02:07,511 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733691727486Initializing all the Stores at 1733691727486Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691727486Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691727487 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691727487Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691727487Cleaning up temporary data from old regions at 1733691727503 (+16 ms)Region opened successfully at 1733691727511 (+8 ms) 2024-12-08T21:02:07,511 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T21:02:07,515 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7f3a47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=229bab1f9d30/172.17.0.2:0 2024-12-08T21:02:07,516 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T21:02:07,516 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T21:02:07,516 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T21:02:07,516 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T21:02:07,517 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-08T21:02:07,517 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-08T21:02:07,517 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T21:02:07,519 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T21:02:07,520 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T21:02:07,545 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T21:02:07,546 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T21:02:07,547 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T21:02:07,554 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T21:02:07,554 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T21:02:07,556 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T21:02:07,562 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T21:02:07,564 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T21:02:07,571 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T21:02:07,575 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T21:02:07,579 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T21:02:07,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:07,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:07,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:07,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,588 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=229bab1f9d30,45245,1733691727145, sessionid=0x100074a1bb60000, setting cluster-up flag (Was=false) 2024-12-08T21:02:07,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,629 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T21:02:07,631 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=229bab1f9d30,45245,1733691727145 2024-12-08T21:02:07,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:07,670 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T21:02:07,672 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=229bab1f9d30,45245,1733691727145 2024-12-08T21:02:07,673 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T21:02:07,674 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:07,674 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T21:02:07,675 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T21:02:07,675 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 229bab1f9d30,45245,1733691727145 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T21:02:07,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:07,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:07,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:07,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:07,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/229bab1f9d30:0, corePoolSize=10, maxPoolSize=10 2024-12-08T21:02:07,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/229bab1f9d30:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:02:07,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,678 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:07,679 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733691757679 2024-12-08T21:02:07,679 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T21:02:07,679 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T21:02:07,679 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T21:02:07,679 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T21:02:07,679 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T21:02:07,679 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:07,679 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T21:02:07,679 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T21:02:07,679 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T21:02:07,679 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,680 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T21:02:07,680 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T21:02:07,681 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T21:02:07,681 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T21:02:07,681 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T21:02:07,681 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.large.0-1733691727681,5,FailOnTimeoutGroup] 2024-12-08T21:02:07,682 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.small.0-1733691727681,5,FailOnTimeoutGroup] 2024-12-08T21:02:07,682 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,682 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T21:02:07,682 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,682 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,694 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T21:02:07,694 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 2024-12-08T21:02:07,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:07,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T21:02:07,705 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T21:02:07,705 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:07,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:07,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T21:02:07,707 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T21:02:07,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:07,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:07,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T21:02:07,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T21:02:07,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:07,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:07,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T21:02:07,714 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T21:02:07,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:07,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:07,714 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T21:02:07,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740 2024-12-08T21:02:07,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740 2024-12-08T21:02:07,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T21:02:07,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T21:02:07,715 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:02:07,716 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T21:02:07,721 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:02:07,722 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65005121, jitterRate=-0.03134821355342865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:02:07,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733691727701Initializing all the Stores at 1733691727701Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691727701Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691727702 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691727702Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691727702Cleaning up temporary data from old regions at 1733691727715 (+13 ms)Region opened successfully at 1733691727722 (+7 ms) 2024-12-08T21:02:07,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T21:02:07,722 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T21:02:07,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T21:02:07,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T21:02:07,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T21:02:07,723 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T21:02:07,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733691727722Disabling compacts and flushes for region at 1733691727722Disabling writes for close at 1733691727723 (+1 ms)Writing region close event to WAL at 1733691727723Closed at 1733691727723 2024-12-08T21:02:07,724 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:07,724 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T21:02:07,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T21:02:07,725 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T21:02:07,726 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T21:02:07,753 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(746): ClusterId : 22cf2d26-7bb4-46b2-9c0c-5322bf0f54c7 2024-12-08T21:02:07,753 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(746): ClusterId : 22cf2d26-7bb4-46b2-9c0c-5322bf0f54c7 2024-12-08T21:02:07,753 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T21:02:07,754 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T21:02:07,781 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T21:02:07,781 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T21:02:07,781 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T21:02:07,781 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T21:02:07,790 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T21:02:07,790 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T21:02:07,790 DEBUG [RS:0;229bab1f9d30:33091 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@199344db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=229bab1f9d30/172.17.0.2:0 2024-12-08T21:02:07,791 DEBUG [RS:1;229bab1f9d30:39131 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355c159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=229bab1f9d30/172.17.0.2:0 2024-12-08T21:02:07,801 DEBUG [RS:0;229bab1f9d30:33091 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;229bab1f9d30:33091 2024-12-08T21:02:07,801 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T21:02:07,801 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T21:02:07,801 DEBUG [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T21:02:07,802 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(2659): reportForDuty to master=229bab1f9d30,45245,1733691727145 with port=33091, startcode=1733691727294 2024-12-08T21:02:07,802 DEBUG [RS:0;229bab1f9d30:33091 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T21:02:07,804 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46757, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T21:02:07,805 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45245 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 229bab1f9d30,33091,1733691727294 2024-12-08T21:02:07,805 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45245 {}] master.ServerManager(517): Registering regionserver=229bab1f9d30,33091,1733691727294 2024-12-08T21:02:07,805 DEBUG [RS:1;229bab1f9d30:39131 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;229bab1f9d30:39131 2024-12-08T21:02:07,805 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T21:02:07,805 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T21:02:07,805 DEBUG [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T21:02:07,806 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(2659): reportForDuty to master=229bab1f9d30,45245,1733691727145 with port=39131, startcode=1733691727326 2024-12-08T21:02:07,806 DEBUG [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 2024-12-08T21:02:07,806 DEBUG [RS:1;229bab1f9d30:39131 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T21:02:07,806 DEBUG [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T21:02:07,806 DEBUG [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T21:02:07,808 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34589, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T21:02:07,809 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45245 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 229bab1f9d30,39131,1733691727326 2024-12-08T21:02:07,809 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45245 {}] master.ServerManager(517): Registering regionserver=229bab1f9d30,39131,1733691727326 2024-12-08T21:02:07,810 DEBUG [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 2024-12-08T21:02:07,810 DEBUG [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T21:02:07,810 DEBUG [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T21:02:07,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:02:07,839 DEBUG [RS:0;229bab1f9d30:33091 {}] zookeeper.ZKUtil(111): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/229bab1f9d30,33091,1733691727294 2024-12-08T21:02:07,839 WARN [RS:0;229bab1f9d30:33091 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:02:07,839 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [229bab1f9d30,33091,1733691727294] 2024-12-08T21:02:07,839 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [229bab1f9d30,39131,1733691727326] 2024-12-08T21:02:07,839 INFO [RS:0;229bab1f9d30:33091 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:07,839 DEBUG [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,33091,1733691727294 2024-12-08T21:02:07,840 DEBUG [RS:1;229bab1f9d30:39131 {}] zookeeper.ZKUtil(111): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/229bab1f9d30,39131,1733691727326 2024-12-08T21:02:07,840 WARN [RS:1;229bab1f9d30:39131 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:02:07,840 INFO [RS:1;229bab1f9d30:39131 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:07,840 DEBUG [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,39131,1733691727326 2024-12-08T21:02:07,846 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T21:02:07,849 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T21:02:07,849 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T21:02:07,851 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T21:02:07,851 INFO [RS:0;229bab1f9d30:33091 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T21:02:07,851 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,851 INFO [RS:1;229bab1f9d30:39131 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T21:02:07,851 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,855 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T21:02:07,855 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T21:02:07,855 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T21:02:07,855 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T21:02:07,855 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,855 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/229bab1f9d30:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/229bab1f9d30:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,856 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,857 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:07,857 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:07,857 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:07,857 DEBUG [RS:0;229bab1f9d30:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:07,857 DEBUG [RS:1;229bab1f9d30:39131 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:07,857 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,39131,1733691727326-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:02:07,857 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,33091,1733691727294-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:02:07,871 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T21:02:07,871 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T21:02:07,871 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,33091,1733691727294-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,871 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,39131,1733691727326-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,871 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,871 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,871 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.Replication(171): 229bab1f9d30,39131,1733691727326 started 2024-12-08T21:02:07,872 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.Replication(171): 229bab1f9d30,33091,1733691727294 started 2024-12-08T21:02:07,876 WARN [229bab1f9d30:45245 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:07,884 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,884 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(1482): Serving as 229bab1f9d30,39131,1733691727326, RpcServer on 229bab1f9d30/172.17.0.2:39131, sessionid=0x100074a1bb60002 2024-12-08T21:02:07,884 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T21:02:07,884 DEBUG [RS:1;229bab1f9d30:39131 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 229bab1f9d30,39131,1733691727326 2024-12-08T21:02:07,884 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,39131,1733691727326' 2024-12-08T21:02:07,884 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T21:02:07,885 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T21:02:07,885 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:07,885 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T21:02:07,885 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T21:02:07,885 DEBUG [RS:1;229bab1f9d30:39131 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 229bab1f9d30,39131,1733691727326 2024-12-08T21:02:07,885 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(1482): Serving as 229bab1f9d30,33091,1733691727294, RpcServer on 229bab1f9d30/172.17.0.2:33091, sessionid=0x100074a1bb60001 2024-12-08T21:02:07,885 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,39131,1733691727326' 2024-12-08T21:02:07,886 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T21:02:07,886 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T21:02:07,886 DEBUG [RS:0;229bab1f9d30:33091 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 229bab1f9d30,33091,1733691727294 2024-12-08T21:02:07,886 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,33091,1733691727294' 2024-12-08T21:02:07,886 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T21:02:07,886 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T21:02:07,886 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T21:02:07,887 DEBUG [RS:1;229bab1f9d30:39131 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T21:02:07,887 INFO [RS:1;229bab1f9d30:39131 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T21:02:07,887 INFO [RS:1;229bab1f9d30:39131 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T21:02:07,887 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T21:02:07,887 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T21:02:07,887 DEBUG [RS:0;229bab1f9d30:33091 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 229bab1f9d30,33091,1733691727294 2024-12-08T21:02:07,887 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,33091,1733691727294' 2024-12-08T21:02:07,887 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T21:02:07,887 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T21:02:07,888 DEBUG [RS:0;229bab1f9d30:33091 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T21:02:07,888 INFO [RS:0;229bab1f9d30:33091 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T21:02:07,888 INFO [RS:0;229bab1f9d30:33091 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T21:02:07,988 INFO [RS:1;229bab1f9d30:39131 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:02:07,989 INFO [RS:1;229bab1f9d30:39131 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=229bab1f9d30%2C39131%2C1733691727326, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,39131,1733691727326, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs, maxLogs=55 2024-12-08T21:02:07,989 INFO [RS:0;229bab1f9d30:33091 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:02:07,991 INFO [RS:0;229bab1f9d30:33091 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=229bab1f9d30%2C33091%2C1733691727294, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,33091,1733691727294, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs, maxLogs=55 2024-12-08T21:02:08,010 INFO [RS:1;229bab1f9d30:39131 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,39131,1733691727326/229bab1f9d30%2C39131%2C1733691727326.1733691727992 2024-12-08T21:02:08,010 DEBUG [RS:1;229bab1f9d30:39131 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T21:02:08,012 INFO [RS:0;229bab1f9d30:33091 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,33091,1733691727294/229bab1f9d30%2C33091%2C1733691727294.1733691727995 2024-12-08T21:02:08,013 DEBUG [RS:0;229bab1f9d30:33091 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T21:02:08,126 DEBUG [229bab1f9d30:45245 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-08T21:02:08,127 DEBUG [229bab1f9d30:45245 {}] balancer.BalancerClusterState(204): Hosts are {229bab1f9d30=0} racks are {/default-rack=0} 2024-12-08T21:02:08,127 DEBUG [229bab1f9d30:45245 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T21:02:08,127 DEBUG [229bab1f9d30:45245 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T21:02:08,128 DEBUG [229bab1f9d30:45245 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T21:02:08,128 DEBUG [229bab1f9d30:45245 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T21:02:08,128 INFO [229bab1f9d30:45245 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T21:02:08,128 INFO [229bab1f9d30:45245 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T21:02:08,128 DEBUG [229bab1f9d30:45245 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T21:02:08,128 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=229bab1f9d30,33091,1733691727294 2024-12-08T21:02:08,129 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 229bab1f9d30,33091,1733691727294, state=OPENING 2024-12-08T21:02:08,162 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T21:02:08,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:08,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:08,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:08,171 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T21:02:08,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=229bab1f9d30,33091,1733691727294}] 2024-12-08T21:02:08,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:08,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:08,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:08,325 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T21:02:08,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59595, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T21:02:08,334 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T21:02:08,334 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:08,335 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T21:02:08,336 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=229bab1f9d30%2C33091%2C1733691727294.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,33091,1733691727294, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs, maxLogs=55 2024-12-08T21:02:08,353 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,33091,1733691727294/229bab1f9d30%2C33091%2C1733691727294.meta.1733691728336.meta 2024-12-08T21:02:08,353 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T21:02:08,353 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:08,354 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T21:02:08,354 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T21:02:08,354 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T21:02:08,354 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T21:02:08,354 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:08,354 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T21:02:08,354 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T21:02:08,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T21:02:08,355 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T21:02:08,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:08,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:08,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T21:02:08,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T21:02:08,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:08,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:08,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T21:02:08,357 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T21:02:08,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:08,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:08,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T21:02:08,358 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T21:02:08,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:08,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:08,358 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T21:02:08,359 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740 2024-12-08T21:02:08,359 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740 2024-12-08T21:02:08,359 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T21:02:08,360 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T21:02:08,360 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:02:08,361 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T21:02:08,361 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72191512, jitterRate=0.0757373571395874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:02:08,361 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T21:02:08,362 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733691728354Writing region info on filesystem at 1733691728354Initializing all the Stores at 1733691728355 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691728355Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691728355Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691728355Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691728355Cleaning up temporary data from old regions at 1733691728360 (+5 ms)Running coprocessor post-open hooks at 1733691728361 (+1 ms)Region opened successfully at 1733691728362 (+1 ms) 2024-12-08T21:02:08,362 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733691728324 2024-12-08T21:02:08,365 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T21:02:08,365 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T21:02:08,365 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,33091,1733691727294 2024-12-08T21:02:08,366 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 229bab1f9d30,33091,1733691727294, state=OPEN 2024-12-08T21:02:08,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:02:08,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:02:08,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:02:08,371 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=229bab1f9d30,33091,1733691727294 2024-12-08T21:02:08,371 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:08,371 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:08,371 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:02:08,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T21:02:08,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=229bab1f9d30,33091,1733691727294 in 200 msec 2024-12-08T21:02:08,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T21:02:08,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 648 msec 2024-12-08T21:02:08,375 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:08,375 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T21:02:08,378 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:02:08,378 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=229bab1f9d30,33091,1733691727294, seqNum=-1] 2024-12-08T21:02:08,378 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:02:08,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60819, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:02:08,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 711 msec 2024-12-08T21:02:08,386 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733691728385, completionTime=-1 2024-12-08T21:02:08,386 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-08T21:02:08,386 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T21:02:08,388 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-08T21:02:08,389 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733691788389 2024-12-08T21:02:08,389 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733691848389 2024-12-08T21:02:08,389 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-08T21:02:08,389 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,45245,1733691727145-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:08,389 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,45245,1733691727145-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:08,390 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,45245,1733691727145-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:08,390 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-229bab1f9d30:45245, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:08,390 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:08,390 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:08,392 DEBUG [master/229bab1f9d30:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T21:02:08,395 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.007sec 2024-12-08T21:02:08,396 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T21:02:08,396 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T21:02:08,396 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T21:02:08,396 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T21:02:08,396 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T21:02:08,396 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,45245,1733691727145-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:02:08,396 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,45245,1733691727145-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T21:02:08,397 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T21:02:08,397 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T21:02:08,397 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,45245,1733691727145-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:08,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa5f19d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:02:08,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 229bab1f9d30,45245,-1 for getting cluster id 2024-12-08T21:02:08,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:02:08,456 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22cf2d26-7bb4-46b2-9c0c-5322bf0f54c7' 2024-12-08T21:02:08,457 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:02:08,457 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22cf2d26-7bb4-46b2-9c0c-5322bf0f54c7" 2024-12-08T21:02:08,458 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79452ce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:02:08,458 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [229bab1f9d30,45245,-1] 2024-12-08T21:02:08,459 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:02:08,459 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:08,461 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:02:08,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22568a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:02:08,462 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:02:08,464 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=229bab1f9d30,33091,1733691727294, seqNum=-1] 2024-12-08T21:02:08,465 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:02:08,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:02:08,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=229bab1f9d30,45245,1733691727145 2024-12-08T21:02:08,470 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:02:08,472 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 229bab1f9d30,45245,1733691727145 2024-12-08T21:02:08,472 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f23bc7 2024-12-08T21:02:08,472 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:02:08,474 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:02:08,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45245 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733691728470', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:02:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45245 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733691728470 2024-12-08T21:02:08,477 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733691728470 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:02:08,478 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:08,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45245 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733691728470" procId is: 4 2024-12-08T21:02:08,479 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733691728470 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:02:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:02:08,491 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733691728470-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9526c0e60ed68259b77166a18021edce, NAME => 'testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733691728470', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 2024-12-08T21:02:08,496 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733691728470-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:08,496 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733691728470-pool-0 {}] regionserver.HRegion(1722): Closing 9526c0e60ed68259b77166a18021edce, disabling compactions & flushes 2024-12-08T21:02:08,496 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733691728470-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:08,496 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733691728470-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:08,496 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733691728470-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. after waiting 0 ms 2024-12-08T21:02:08,496 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733691728470-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:08,496 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733691728470-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:08,496 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733691728470-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9526c0e60ed68259b77166a18021edce: Waiting for close lock at 1733691728496Disabling compacts and flushes for region at 1733691728496Disabling writes for close at 1733691728496Writing region close event to WAL at 1733691728496Closed at 1733691728496 2024-12-08T21:02:08,497 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733691728470 execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:02:08,497 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733691728497"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691728497"}]},"ts":"1733691728497"} 2024-12-08T21:02:08,500 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:02:08,500 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733691728470 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:02:08,501 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733691728470","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691728500"}]},"ts":"1733691728500"} 2024-12-08T21:02:08,503 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733691728470, state=ENABLING in hbase:meta 2024-12-08T21:02:08,503 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {229bab1f9d30=0} racks are {/default-rack=0} 2024-12-08T21:02:08,504 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T21:02:08,504 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T21:02:08,504 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T21:02:08,504 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T21:02:08,504 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T21:02:08,504 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T21:02:08,504 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T21:02:08,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN}] 2024-12-08T21:02:08,505 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN 2024-12-08T21:02:08,506 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN; state=OFFLINE, location=229bab1f9d30,39131,1733691727326; forceNewPlan=false, retain=false 2024-12-08T21:02:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:02:08,657 INFO [229bab1f9d30:45245 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T21:02:08,657 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9526c0e60ed68259b77166a18021edce, regionState=OPENING, regionLocation=229bab1f9d30,39131,1733691727326 2024-12-08T21:02:08,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN because future has completed 2024-12-08T21:02:08,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9526c0e60ed68259b77166a18021edce, server=229bab1f9d30,39131,1733691727326}] 2024-12-08T21:02:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:02:08,817 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T21:02:08,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45675, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T21:02:08,822 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:08,822 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9526c0e60ed68259b77166a18021edce, NAME => 'testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:08,823 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733691728470 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,823 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:08,823 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,823 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,824 INFO [StoreOpener-9526c0e60ed68259b77166a18021edce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,827 INFO [StoreOpener-9526c0e60ed68259b77166a18021edce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9526c0e60ed68259b77166a18021edce columnFamilyName family 2024-12-08T21:02:08,827 DEBUG [StoreOpener-9526c0e60ed68259b77166a18021edce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:08,827 INFO [StoreOpener-9526c0e60ed68259b77166a18021edce-1 {}] regionserver.HStore(327): Store=9526c0e60ed68259b77166a18021edce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:08,827 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,828 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,828 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,828 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,828 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,829 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,834 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:02:08,834 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9526c0e60ed68259b77166a18021edce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65296212, jitterRate=-0.027010619640350342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:02:08,835 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:08,835 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9526c0e60ed68259b77166a18021edce: Running coprocessor pre-open hook at 1733691728823Writing region info on filesystem at 1733691728823Initializing all the Stores at 1733691728823Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691728823Cleaning up temporary data from old regions at 1733691728828 (+5 ms)Running coprocessor post-open hooks at 1733691728835 (+7 ms)Region opened successfully at 1733691728835 2024-12-08T21:02:08,836 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce., pid=6, masterSystemTime=1733691728817 2024-12-08T21:02:08,839 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:08,839 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:08,839 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9526c0e60ed68259b77166a18021edce, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,39131,1733691727326 2024-12-08T21:02:08,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9526c0e60ed68259b77166a18021edce, server=229bab1f9d30,39131,1733691727326 because future has completed 2024-12-08T21:02:08,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T21:02:08,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9526c0e60ed68259b77166a18021edce, server=229bab1f9d30,39131,1733691727326 in 180 msec 2024-12-08T21:02:08,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T21:02:08,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN in 341 msec 2024-12-08T21:02:08,848 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733691728470 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:02:08,848 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733691728470","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691728848"}]},"ts":"1733691728848"} 2024-12-08T21:02:08,850 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733691728470, state=ENABLED in hbase:meta 2024-12-08T21:02:08,851 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733691728470 execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:02:08,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733691728470 in 376 msec 2024-12-08T21:02:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:02:09,109 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733691728470 completed 2024-12-08T21:02:09,109 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733691728470 get assigned. Timeout = 60000ms 2024-12-08T21:02:09,109 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:02:09,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733691728470 assigned to meta. Checking AM states. 2024-12-08T21:02:09,112 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:02:09,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733691728470 assigned. 2024-12-08T21:02:09,117 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733691728470', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce., hostname=229bab1f9d30,39131,1733691727326, seqNum=2] 2024-12-08T21:02:09,117 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:02:09,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:02:09,125 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9526c0e60ed68259b77166a18021edce 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-08T21:02:09,135 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 365ed69aa34645cbafcd5f8fc1a854ca 2024-12-08T21:02:09,153 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/.tmp/family/365ed69aa34645cbafcd5f8fc1a854ca is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-08T21:02:09,154 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/.tmp/family/365ed69aa34645cbafcd5f8fc1a854ca 2024-12-08T21:02:09,157 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/.tmp/family/365ed69aa34645cbafcd5f8fc1a854ca as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca 2024-12-08T21:02:09,159 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca, delay=978 ms 2024-12-08T21:02:09,160 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca, entries=2, sequenceid=6, filesize=4.9 K 2024-12-08T21:02:09,161 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 9526c0e60ed68259b77166a18021edce in 36ms, sequenceid=6, compaction requested=false 2024-12-08T21:02:09,161 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733691728470' 2024-12-08T21:02:09,162 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9526c0e60ed68259b77166a18021edce: 2024-12-08T21:02:09,399 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:02:10,138 DEBUG [hfile-prefetch-1733691724971 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 365ed69aa34645cbafcd5f8fc1a854ca_0. 2024-12-08T21:02:10,139 DEBUG [hfile-prefetch-1733691724971 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 365ed69aa34645cbafcd5f8fc1a854ca_117. 2024-12-08T21:02:10,139 DEBUG [hfile-prefetch-1733691724971 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca, with total blocks 2, and data blocks 1 2024-12-08T21:02:10,139 DEBUG [hfile-prefetch-1733691724971 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-08T21:02:10,139 DEBUG [hfile-prefetch-1733691724971 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-08T21:02:10,139 DEBUG [hfile-prefetch-1733691724971 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca. Will sleep 100ms and try the verification again. 2024-12-08T21:02:10,162 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;229bab1f9d30:39131,5,FailOnTimeoutGroup] 2024-12-08T21:02:10,163 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '229bab1f9d30,39131,1733691727326' ***** 2024-12-08T21:02:10,163 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-08T21:02:10,163 INFO [RS:1;229bab1f9d30:39131 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T21:02:10,214 INFO [RS:1;229bab1f9d30:39131 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.persistence 2024-12-08T21:02:10,214 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T21:02:10,214 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T21:02:10,214 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T21:02:10,215 INFO [RS:1;229bab1f9d30:39131 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T21:02:10,215 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T21:02:10,225 DEBUG [RS:1;229bab1f9d30:39131 {}] bucket.BucketCache(1487): Thread RS:1;229bab1f9d30:39131 started persisting bucket cache to file 2024-12-08T21:02:10,225 DEBUG [RS:1;229bab1f9d30:39131 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T21:02:10,225 DEBUG [RS:1;229bab1f9d30:39131 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T21:02:10,228 DEBUG [RS:1;229bab1f9d30:39131 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-08T21:02:10,228 DEBUG [RS:1;229bab1f9d30:39131 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733691730225 2024-12-08T21:02:10,228 DEBUG [RS:1;229bab1f9d30:39131 {}] bucket.BucketCache(1509): Thread RS:1;229bab1f9d30:39131 finished persisting bucket cache to file, renaming 2024-12-08T21:02:10,228 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T21:02:10,228 INFO [RS:1;229bab1f9d30:39131 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T21:02:10,228 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T21:02:10,228 INFO [RS:1;229bab1f9d30:39131 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T21:02:10,228 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(3091): Received CLOSE for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,229 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(959): stopping server 229bab1f9d30,39131,1733691727326 2024-12-08T21:02:10,229 INFO [RS:1;229bab1f9d30:39131 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:02:10,229 INFO [RS:1;229bab1f9d30:39131 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;229bab1f9d30:39131. 2024-12-08T21:02:10,229 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T21:02:10,229 DEBUG [RS:1;229bab1f9d30:39131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:02:10,229 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9526c0e60ed68259b77166a18021edce, disabling compactions & flushes 2024-12-08T21:02:10,229 DEBUG [RS:1;229bab1f9d30:39131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:10,229 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:10,229 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:10,229 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T21:02:10,229 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. after waiting 0 ms 2024-12-08T21:02:10,229 DEBUG [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(1325): Online Regions={9526c0e60ed68259b77166a18021edce=testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce.} 2024-12-08T21:02:10,229 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:10,229 DEBUG [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(1351): Waiting on 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,230 DEBUG [StoreFileCloser-9526c0e60ed68259b77166a18021edce-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca 2024-12-08T21:02:10,230 WARN [hfile-prefetch-1733691724971 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-08T21:02:10,230 DEBUG [hfile-prefetch-1733691724971 {}] hfile.PrefetchExecutor(121): Prefetch completed for 365ed69aa34645cbafcd5f8fc1a854ca 2024-12-08T21:02:10,234 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-08T21:02:10,234 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:10,234 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9526c0e60ed68259b77166a18021edce: Waiting for close lock at 1733691730229Running coprocessor pre-close hooks at 1733691730229Disabling compacts and flushes for region at 1733691730229Disabling writes for close at 1733691730229Writing region close event to WAL at 1733691730230 (+1 ms)Running coprocessor post-close hooks at 1733691730234 (+4 ms)Closed at 1733691730234 2024-12-08T21:02:10,234 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:10,260 INFO [regionserver/229bab1f9d30:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:10,309 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-08T21:02:10,309 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-08T21:02:10,310 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T21:02:10,310 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T21:02:10,316 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T21:02:10,316 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733691730309 2024-12-08T21:02:10,316 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-08T21:02:10,430 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(976): stopping server 229bab1f9d30,39131,1733691727326; all regions closed. 2024-12-08T21:02:10,432 DEBUG [RS:1;229bab1f9d30:39131 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs 2024-12-08T21:02:10,432 INFO [RS:1;229bab1f9d30:39131 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 229bab1f9d30%2C39131%2C1733691727326:(num 1733691727992) 2024-12-08T21:02:10,432 DEBUG [RS:1;229bab1f9d30:39131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:10,432 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:10,432 INFO [RS:1;229bab1f9d30:39131 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:02:10,432 INFO [RS:1;229bab1f9d30:39131 {}] hbase.ChoreService(370): Chore service for: regionserver/229bab1f9d30:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T21:02:10,433 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T21:02:10,433 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T21:02:10,433 INFO [regionserver/229bab1f9d30:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:02:10,433 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T21:02:10,433 INFO [RS:1;229bab1f9d30:39131 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:02:10,433 INFO [RS:1;229bab1f9d30:39131 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39131 2024-12-08T21:02:10,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/229bab1f9d30,39131,1733691727326 2024-12-08T21:02:10,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:02:10,462 INFO [RS:1;229bab1f9d30:39131 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:02:10,463 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [229bab1f9d30,39131,1733691727326] 2024-12-08T21:02:10,479 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/229bab1f9d30,39131,1733691727326 already deleted, retry=false 2024-12-08T21:02:10,479 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 229bab1f9d30,39131,1733691727326 on 229bab1f9d30,45245,1733691727145 2024-12-08T21:02:10,486 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 229bab1f9d30,39131,1733691727326, splitWal=true, meta=false 2024-12-08T21:02:10,489 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 229bab1f9d30,39131,1733691727326, splitWal=true, meta=false 2024-12-08T21:02:10,489 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 229bab1f9d30,39131,1733691727326 (carryingMeta=false) 229bab1f9d30,39131,1733691727326/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@311f03ba[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-08T21:02:10,491 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 229bab1f9d30,39131,1733691727326 had 1 regions 2024-12-08T21:02:10,492 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 229bab1f9d30,39131,1733691727326, splitWal=true, meta=false, isMeta: false 2024-12-08T21:02:10,493 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,39131,1733691727326-splitting 2024-12-08T21:02:10,493 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,39131,1733691727326-splitting dir is empty, no logs to split. 2024-12-08T21:02:10,493 INFO [PEWorker-1 {}] master.SplitWALManager(105): 229bab1f9d30,39131,1733691727326 WAL count=0, meta=false 2024-12-08T21:02:10,496 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,39131,1733691727326-splitting dir is empty, no logs to split. 2024-12-08T21:02:10,496 INFO [PEWorker-1 {}] master.SplitWALManager(105): 229bab1f9d30,39131,1733691727326 WAL count=0, meta=false 2024-12-08T21:02:10,496 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 229bab1f9d30,39131,1733691727326 WAL splitting is done? wals=0, meta=false 2024-12-08T21:02:10,496 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 229bab1f9d30,39131,1733691727326 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,39131,1733691727326-splitting does not exist 2024-12-08T21:02:10,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN}] 2024-12-08T21:02:10,498 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN 2024-12-08T21:02:10,499 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-08T21:02:10,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:10,571 INFO [RS:1;229bab1f9d30:39131 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:02:10,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39131-0x100074a1bb60002, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:10,571 INFO [RS:1;229bab1f9d30:39131 {}] regionserver.HRegionServer(1031): Exiting; stopping=229bab1f9d30,39131,1733691727326; zookeeper connection closed. 2024-12-08T21:02:10,572 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2ca94efb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2ca94efb 2024-12-08T21:02:10,649 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=9526c0e60ed68259b77166a18021edce, regionState=OPENING, regionLocation=229bab1f9d30,33091,1733691727294 2024-12-08T21:02:10,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN because future has completed 2024-12-08T21:02:10,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9526c0e60ed68259b77166a18021edce, server=229bab1f9d30,33091,1733691727294}] 2024-12-08T21:02:10,682 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/229bab1f9d30:0 server-side Connection retries=45 2024-12-08T21:02:10,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:10,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:10,683 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:02:10,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:10,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:02:10,683 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T21:02:10,683 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:02:10,684 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38765 2024-12-08T21:02:10,685 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38765 connecting to ZooKeeper ensemble=127.0.0.1:60212 2024-12-08T21:02:10,685 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:10,685 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:10,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387650x0, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:02:10,713 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38765-0x100074a1bb60003 connected 2024-12-08T21:02:10,713 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-08T21:02:10,713 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-08T21:02:10,714 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T21:02:10,714 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.cache 2024-12-08T21:02:10,714 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T21:02:10,714 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T21:02:10,714 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T21:02:10,715 INFO [Thread-199 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T21:02:10,715 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T21:02:10,715 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T21:02:10,716 INFO [Thread-199 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-08T21:02:10,716 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T21:02:10,716 INFO [Thread-199 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-08T21:02:10,718 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:02:10,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38765 2024-12-08T21:02:10,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38765 2024-12-08T21:02:10,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38765 2024-12-08T21:02:10,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38765 2024-12-08T21:02:10,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38765 2024-12-08T21:02:10,722 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(746): ClusterId : 22cf2d26-7bb4-46b2-9c0c-5322bf0f54c7 2024-12-08T21:02:10,722 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T21:02:10,724 DEBUG [Thread-199 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-08T21:02:10,725 INFO [Thread-199 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T21:02:10,725 INFO [Thread-199 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-08T21:02:10,725 INFO [Thread-199 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.persistence is complete. 2024-12-08T21:02:10,730 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T21:02:10,730 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T21:02:10,738 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T21:02:10,738 DEBUG [RS:2;229bab1f9d30:38765 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e8a69d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=229bab1f9d30/172.17.0.2:0 2024-12-08T21:02:10,748 DEBUG [RS:2;229bab1f9d30:38765 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;229bab1f9d30:38765 2024-12-08T21:02:10,748 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T21:02:10,748 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T21:02:10,748 DEBUG [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T21:02:10,749 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(2659): reportForDuty to master=229bab1f9d30,45245,1733691727145 with port=38765, startcode=1733691730682 2024-12-08T21:02:10,749 DEBUG [RS:2;229bab1f9d30:38765 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T21:02:10,751 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38567, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T21:02:10,752 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45245 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 229bab1f9d30,38765,1733691730682 2024-12-08T21:02:10,752 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45245 {}] master.ServerManager(517): Registering regionserver=229bab1f9d30,38765,1733691730682 2024-12-08T21:02:10,753 DEBUG [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200 2024-12-08T21:02:10,753 DEBUG [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T21:02:10,753 DEBUG [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T21:02:10,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:02:10,762 DEBUG [RS:2;229bab1f9d30:38765 {}] zookeeper.ZKUtil(111): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/229bab1f9d30,38765,1733691730682 2024-12-08T21:02:10,763 WARN [RS:2;229bab1f9d30:38765 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:02:10,763 INFO [RS:2;229bab1f9d30:38765 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:10,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [229bab1f9d30,38765,1733691730682] 2024-12-08T21:02:10,763 DEBUG [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,38765,1733691730682 2024-12-08T21:02:10,768 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T21:02:10,769 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T21:02:10,770 INFO [RS:2;229bab1f9d30:38765 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T21:02:10,770 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,770 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T21:02:10,771 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T21:02:10,771 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/229bab1f9d30:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,771 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,772 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:10,772 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:10,772 DEBUG [RS:2;229bab1f9d30:38765 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:10,772 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,772 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,772 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,772 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,772 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,772 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,38765,1733691730682-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:02:10,787 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T21:02:10,787 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,38765,1733691730682-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,788 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,788 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.Replication(171): 229bab1f9d30,38765,1733691730682 started 2024-12-08T21:02:10,800 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:10,800 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(1482): Serving as 229bab1f9d30,38765,1733691730682, RpcServer on 229bab1f9d30/172.17.0.2:38765, sessionid=0x100074a1bb60003 2024-12-08T21:02:10,801 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T21:02:10,801 DEBUG [RS:2;229bab1f9d30:38765 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 229bab1f9d30,38765,1733691730682 2024-12-08T21:02:10,801 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,38765,1733691730682' 2024-12-08T21:02:10,801 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T21:02:10,801 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T21:02:10,802 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T21:02:10,802 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T21:02:10,802 DEBUG [RS:2;229bab1f9d30:38765 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 229bab1f9d30,38765,1733691730682 2024-12-08T21:02:10,802 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,38765,1733691730682' 2024-12-08T21:02:10,802 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T21:02:10,802 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T21:02:10,803 DEBUG [RS:2;229bab1f9d30:38765 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T21:02:10,803 INFO [RS:2;229bab1f9d30:38765 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T21:02:10,803 INFO [RS:2;229bab1f9d30:38765 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T21:02:10,807 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:10,808 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9526c0e60ed68259b77166a18021edce, NAME => 'testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:10,808 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733691728470 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,808 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:10,808 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,808 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,809 INFO [StoreOpener-9526c0e60ed68259b77166a18021edce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,810 INFO [StoreOpener-9526c0e60ed68259b77166a18021edce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9526c0e60ed68259b77166a18021edce columnFamilyName family 2024-12-08T21:02:10,810 DEBUG [StoreOpener-9526c0e60ed68259b77166a18021edce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:10,811 DEBUG [StoreFileOpener-9526c0e60ed68259b77166a18021edce-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca, delay=931 ms 2024-12-08T21:02:10,812 DEBUG [StoreOpener-9526c0e60ed68259b77166a18021edce-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca 2024-12-08T21:02:10,812 INFO [StoreOpener-9526c0e60ed68259b77166a18021edce-1 {}] regionserver.HStore(327): Store=9526c0e60ed68259b77166a18021edce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:10,813 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,814 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,814 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,814 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,814 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,816 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,816 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9526c0e60ed68259b77166a18021edce; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74623273, jitterRate=0.11197341978549957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:02:10,816 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:10,817 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9526c0e60ed68259b77166a18021edce: Running coprocessor pre-open hook at 1733691730809Writing region info on filesystem at 1733691730809Initializing all the Stores at 1733691730809Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691730809Cleaning up temporary data from old regions at 1733691730814 (+5 ms)Running coprocessor post-open hooks at 1733691730816 (+2 ms)Region opened successfully at 1733691730817 (+1 ms) 2024-12-08T21:02:10,818 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce., pid=9, masterSystemTime=1733691730805 2024-12-08T21:02:10,821 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:10,821 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:10,821 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=9526c0e60ed68259b77166a18021edce, regionState=OPEN, openSeqNum=10, regionLocation=229bab1f9d30,33091,1733691727294 2024-12-08T21:02:10,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9526c0e60ed68259b77166a18021edce, server=229bab1f9d30,33091,1733691727294 because future has completed 2024-12-08T21:02:10,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-08T21:02:10,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 9526c0e60ed68259b77166a18021edce, server=229bab1f9d30,33091,1733691727294 in 171 msec 2024-12-08T21:02:10,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T21:02:10,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733691728470, region=9526c0e60ed68259b77166a18021edce, ASSIGN in 329 msec 2024-12-08T21:02:10,828 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 229bab1f9d30,39131,1733691727326 after splitting done 2024-12-08T21:02:10,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 229bab1f9d30,39131,1733691727326, splitWal=true, meta=false in 348 msec 2024-12-08T21:02:10,904 INFO [RS:2;229bab1f9d30:38765 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:02:10,905 INFO [RS:2;229bab1f9d30:38765 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=229bab1f9d30%2C38765%2C1733691730682, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,38765,1733691730682, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs, maxLogs=55 2024-12-08T21:02:10,925 INFO [RS:2;229bab1f9d30:38765 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/WALs/229bab1f9d30,38765,1733691730682/229bab1f9d30%2C38765%2C1733691730682.1733691730905 2024-12-08T21:02:10,925 DEBUG [RS:2;229bab1f9d30:38765 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T21:02:11,301 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T21:02:11,301 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:02:11,301 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:02:11,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:11,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:11,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:02:11,302 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T21:02:11,302 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1523998881, stopped=false 2024-12-08T21:02:11,303 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=229bab1f9d30,45245,1733691727145 2024-12-08T21:02:11,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:11,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:11,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:11,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:11,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:11,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:11,346 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T21:02:11,346 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:02:11,347 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:11,347 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:11,348 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:02:11,348 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:11,348 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(878): Closing user regions 2024-12-08T21:02:11,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:11,348 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(3091): Received CLOSE for 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:11,349 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '229bab1f9d30,33091,1733691727294' ***** 2024-12-08T21:02:11,349 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T21:02:11,349 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '229bab1f9d30,38765,1733691730682' ***** 2024-12-08T21:02:11,349 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T21:02:11,349 INFO [RS:2;229bab1f9d30:38765 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T21:02:11,349 INFO [RS:2;229bab1f9d30:38765 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.persistence 2024-12-08T21:02:11,349 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T21:02:11,349 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T21:02:11,349 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T21:02:11,349 INFO [RS:2;229bab1f9d30:38765 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T21:02:11,349 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T21:02:11,350 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9526c0e60ed68259b77166a18021edce, disabling compactions & flushes 2024-12-08T21:02:11,350 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T21:02:11,350 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:11,350 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:11,350 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. after waiting 0 ms 2024-12-08T21:02:11,350 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:11,350 DEBUG [RS:2;229bab1f9d30:38765 {}] bucket.BucketCache(1487): Thread RS:2;229bab1f9d30:38765 started persisting bucket cache to file 2024-12-08T21:02:11,350 DEBUG [RS:2;229bab1f9d30:38765 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T21:02:11,350 DEBUG [RS:2;229bab1f9d30:38765 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T21:02:11,350 DEBUG [StoreFileCloser-9526c0e60ed68259b77166a18021edce-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/family/365ed69aa34645cbafcd5f8fc1a854ca 2024-12-08T21:02:11,351 INFO [RS:0;229bab1f9d30:33091 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T21:02:11,351 INFO [RS:0;229bab1f9d30:33091 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/613b6fe9-1015-04eb-1788-19defd14fa7e/bucket.persistence 2024-12-08T21:02:11,351 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T21:02:11,351 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T21:02:11,351 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T21:02:11,352 INFO [RS:0;229bab1f9d30:33091 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T21:02:11,352 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T21:02:11,354 DEBUG [RS:2;229bab1f9d30:38765 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T21:02:11,354 DEBUG [RS:2;229bab1f9d30:38765 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733691731350 2024-12-08T21:02:11,354 DEBUG [RS:2;229bab1f9d30:38765 {}] bucket.BucketCache(1509): Thread RS:2;229bab1f9d30:38765 finished persisting bucket cache to file, renaming 2024-12-08T21:02:11,354 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T21:02:11,354 INFO [RS:2;229bab1f9d30:38765 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T21:02:11,354 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T21:02:11,354 INFO [RS:2;229bab1f9d30:38765 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T21:02:11,354 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(959): stopping server 229bab1f9d30,38765,1733691730682 2024-12-08T21:02:11,354 INFO [RS:2;229bab1f9d30:38765 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:02:11,354 INFO [RS:2;229bab1f9d30:38765 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;229bab1f9d30:38765. 2024-12-08T21:02:11,354 DEBUG [RS:2;229bab1f9d30:38765 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:02:11,354 DEBUG [RS:2;229bab1f9d30:38765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:11,354 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(976): stopping server 229bab1f9d30,38765,1733691730682; all regions closed. 2024-12-08T21:02:11,354 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/default/testBlockEvictionOnGracefulStop1733691728470/9526c0e60ed68259b77166a18021edce/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-08T21:02:11,355 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:11,355 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9526c0e60ed68259b77166a18021edce: Waiting for close lock at 1733691731349Running coprocessor pre-close hooks at 1733691731350 (+1 ms)Disabling compacts and flushes for region at 1733691731350Disabling writes for close at 1733691731350Writing region close event to WAL at 1733691731351 (+1 ms)Running coprocessor post-close hooks at 1733691731355 (+4 ms)Closed at 1733691731355 2024-12-08T21:02:11,356 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45245 {}] assignment.AssignmentManager(1314): RegionServer CLOSED 9526c0e60ed68259b77166a18021edce 2024-12-08T21:02:11,356 DEBUG [RS:2;229bab1f9d30:38765 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs 2024-12-08T21:02:11,356 INFO [RS:2;229bab1f9d30:38765 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 229bab1f9d30%2C38765%2C1733691730682:(num 1733691730905) 2024-12-08T21:02:11,357 DEBUG [RS:2;229bab1f9d30:38765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:11,357 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:11,357 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce. 2024-12-08T21:02:11,357 INFO [RS:2;229bab1f9d30:38765 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:02:11,357 INFO [RS:2;229bab1f9d30:38765 {}] hbase.ChoreService(370): Chore service for: regionserver/229bab1f9d30:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T21:02:11,357 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T21:02:11,357 INFO [regionserver/229bab1f9d30:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:02:11,357 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T21:02:11,357 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T21:02:11,357 INFO [RS:2;229bab1f9d30:38765 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:02:11,357 INFO [RS:2;229bab1f9d30:38765 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38765 2024-12-08T21:02:11,361 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T21:02:11,361 INFO [regionserver/229bab1f9d30:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:11,362 DEBUG [RS:0;229bab1f9d30:33091 {}] bucket.BucketCache(1487): Thread RS:0;229bab1f9d30:33091 started persisting bucket cache to file 2024-12-08T21:02:11,362 DEBUG [RS:0;229bab1f9d30:33091 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T21:02:11,362 DEBUG [RS:0;229bab1f9d30:33091 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T21:02:11,365 DEBUG [RS:0;229bab1f9d30:33091 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T21:02:11,365 DEBUG [RS:0;229bab1f9d30:33091 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733691731362 2024-12-08T21:02:11,365 DEBUG [RS:0;229bab1f9d30:33091 {}] bucket.BucketCache(1509): Thread RS:0;229bab1f9d30:33091 finished persisting bucket cache to file, renaming 2024-12-08T21:02:11,365 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T21:02:11,365 INFO [RS:0;229bab1f9d30:33091 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T21:02:11,365 INFO [RS:0;229bab1f9d30:33091 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T21:02:11,365 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(959): stopping server 229bab1f9d30,33091,1733691727294 2024-12-08T21:02:11,365 INFO [RS:0;229bab1f9d30:33091 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:02:11,366 INFO [RS:0;229bab1f9d30:33091 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;229bab1f9d30:33091. 2024-12-08T21:02:11,366 DEBUG [RS:0;229bab1f9d30:33091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:02:11,366 DEBUG [RS:0;229bab1f9d30:33091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:11,366 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T21:02:11,366 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T21:02:11,366 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T21:02:11,366 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T21:02:11,366 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T21:02:11,366 DEBUG [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T21:02:11,367 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T21:02:11,367 DEBUG [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:02:11,367 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T21:02:11,367 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T21:02:11,367 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T21:02:11,367 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T21:02:11,367 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T21:02:11,367 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-08T21:02:11,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:02:11,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/229bab1f9d30,38765,1733691730682 2024-12-08T21:02:11,372 INFO [RS:2;229bab1f9d30:38765 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:02:11,375 INFO [regionserver/229bab1f9d30:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:11,377 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6cbe28e8eee6490c9f51e8368e6c55ae 2024-12-08T21:02:11,387 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [229bab1f9d30,38765,1733691730682] 2024-12-08T21:02:11,391 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/info/6cbe28e8eee6490c9f51e8368e6c55ae is 209, key is testBlockEvictionOnGracefulStop1733691728470,,1733691728475.9526c0e60ed68259b77166a18021edce./info:regioninfo/1733691730821/Put/seqid=0 2024-12-08T21:02:11,392 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/info/6cbe28e8eee6490c9f51e8368e6c55ae 2024-12-08T21:02:11,395 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/229bab1f9d30,38765,1733691730682 already deleted, retry=false 2024-12-08T21:02:11,395 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 229bab1f9d30,38765,1733691730682 expired; onlineServers=1 2024-12-08T21:02:11,399 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 286e7c6b8be84c50baaed53e54bd5f2d 2024-12-08T21:02:11,410 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/ns/286e7c6b8be84c50baaed53e54bd5f2d is 43, key is default/ns:d/1733691728381/Put/seqid=0 2024-12-08T21:02:11,411 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/ns/286e7c6b8be84c50baaed53e54bd5f2d 2024-12-08T21:02:11,419 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 375b4706b77647ab9c316d211e616769 2024-12-08T21:02:11,430 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/table/375b4706b77647ab9c316d211e616769 is 80, key is testBlockEvictionOnGracefulStop1733691728470/table:state/1733691728848/Put/seqid=0 2024-12-08T21:02:11,430 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/table/375b4706b77647ab9c316d211e616769 2024-12-08T21:02:11,432 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/info/6cbe28e8eee6490c9f51e8368e6c55ae as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/info/6cbe28e8eee6490c9f51e8368e6c55ae 2024-12-08T21:02:11,433 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/info/6cbe28e8eee6490c9f51e8368e6c55ae, delay=987 ms 2024-12-08T21:02:11,434 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/info/6cbe28e8eee6490c9f51e8368e6c55ae, entries=14, sequenceid=13, filesize=7.5 K 2024-12-08T21:02:11,435 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/ns/286e7c6b8be84c50baaed53e54bd5f2d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/ns/286e7c6b8be84c50baaed53e54bd5f2d 2024-12-08T21:02:11,436 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/ns/286e7c6b8be84c50baaed53e54bd5f2d, delay=980 ms 2024-12-08T21:02:11,437 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/ns/286e7c6b8be84c50baaed53e54bd5f2d, entries=2, sequenceid=13, filesize=5.0 K 2024-12-08T21:02:11,437 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/.tmp/table/375b4706b77647ab9c316d211e616769 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/table/375b4706b77647ab9c316d211e616769 2024-12-08T21:02:11,438 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/table/375b4706b77647ab9c316d211e616769, delay=941 ms 2024-12-08T21:02:11,440 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/table/375b4706b77647ab9c316d211e616769, entries=2, sequenceid=13, filesize=5.3 K 2024-12-08T21:02:11,441 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=13, compaction requested=false 2024-12-08T21:02:11,441 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/info/6cbe28e8eee6490c9f51e8368e6c55ae 2024-12-08T21:02:11,442 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/ns/286e7c6b8be84c50baaed53e54bd5f2d 2024-12-08T21:02:11,442 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/table/375b4706b77647ab9c316d211e616769 2024-12-08T21:02:11,446 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-08T21:02:11,447 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T21:02:11,447 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T21:02:11,447 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733691731367Running coprocessor pre-close hooks at 1733691731367Disabling compacts and flushes for region at 1733691731367Disabling writes for close at 1733691731367Obtaining lock to block concurrent updates at 1733691731367Preparing flush snapshotting stores in 1588230740 at 1733691731367Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733691731367Flushing stores of hbase:meta,,1.1588230740 at 1733691731368 (+1 ms)Flushing 1588230740/info: creating writer at 1733691731368Flushing 1588230740/info: appending metadata at 1733691731390 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733691731390Flushing 1588230740/ns: creating writer at 1733691731394 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733691731410 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733691731410Flushing 1588230740/table: creating writer at 1733691731413 (+3 ms)Flushing 1588230740/table: appending metadata at 1733691731429 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733691731429Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fd97dd8: reopening flushed file at 1733691731432 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@535ec064: reopening flushed file at 1733691731435 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59ccdf5f: reopening flushed file at 1733691731437 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=13, compaction requested=false at 1733691731441 (+4 ms)Writing region close event to WAL at 1733691731443 (+2 ms)Running coprocessor post-close hooks at 1733691731447 (+4 ms)Closed at 1733691731447 2024-12-08T21:02:11,447 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T21:02:11,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:11,487 INFO [RS:2;229bab1f9d30:38765 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:02:11,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38765-0x100074a1bb60003, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:11,487 INFO [RS:2;229bab1f9d30:38765 {}] regionserver.HRegionServer(1031): Exiting; stopping=229bab1f9d30,38765,1733691730682; zookeeper connection closed. 2024-12-08T21:02:11,487 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3cb47b09 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3cb47b09 2024-12-08T21:02:11,512 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T21:02:11,513 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T21:02:11,567 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(976): stopping server 229bab1f9d30,33091,1733691727294; all regions closed. 2024-12-08T21:02:11,570 DEBUG [RS:0;229bab1f9d30:33091 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs 2024-12-08T21:02:11,570 INFO [RS:0;229bab1f9d30:33091 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 229bab1f9d30%2C33091%2C1733691727294.meta:.meta(num 1733691728336) 2024-12-08T21:02:11,572 DEBUG [RS:0;229bab1f9d30:33091 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/oldWALs 2024-12-08T21:02:11,572 INFO [RS:0;229bab1f9d30:33091 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 229bab1f9d30%2C33091%2C1733691727294:(num 1733691727995) 2024-12-08T21:02:11,572 DEBUG [RS:0;229bab1f9d30:33091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:02:11,572 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:02:11,572 INFO [RS:0;229bab1f9d30:33091 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:02:11,573 INFO [RS:0;229bab1f9d30:33091 {}] hbase.ChoreService(370): Chore service for: regionserver/229bab1f9d30:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T21:02:11,573 INFO [RS:0;229bab1f9d30:33091 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:02:11,573 INFO [regionserver/229bab1f9d30:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:02:11,573 INFO [RS:0;229bab1f9d30:33091 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33091 2024-12-08T21:02:11,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:02:11,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/229bab1f9d30,33091,1733691727294 2024-12-08T21:02:11,587 INFO [RS:0;229bab1f9d30:33091 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:02:11,595 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [229bab1f9d30,33091,1733691727294] 2024-12-08T21:02:11,603 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/229bab1f9d30,33091,1733691727294 already deleted, retry=false 2024-12-08T21:02:11,603 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 229bab1f9d30,33091,1733691727294 expired; onlineServers=0 2024-12-08T21:02:11,603 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '229bab1f9d30,45245,1733691727145' ***** 2024-12-08T21:02:11,603 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T21:02:11,604 INFO [M:0;229bab1f9d30:45245 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:02:11,604 INFO [M:0;229bab1f9d30:45245 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:02:11,604 DEBUG [M:0;229bab1f9d30:45245 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T21:02:11,604 DEBUG [M:0;229bab1f9d30:45245 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T21:02:11,604 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T21:02:11,604 DEBUG [master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.large.0-1733691727681 {}] cleaner.HFileCleaner(306): Exit Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.large.0-1733691727681,5,FailOnTimeoutGroup] 2024-12-08T21:02:11,604 DEBUG [master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.small.0-1733691727681 {}] cleaner.HFileCleaner(306): Exit Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.small.0-1733691727681,5,FailOnTimeoutGroup] 2024-12-08T21:02:11,604 INFO [M:0;229bab1f9d30:45245 {}] hbase.ChoreService(370): Chore service for: master/229bab1f9d30:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T21:02:11,605 INFO [M:0;229bab1f9d30:45245 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:02:11,605 DEBUG [M:0;229bab1f9d30:45245 {}] master.HMaster(1795): Stopping service threads 2024-12-08T21:02:11,605 INFO [M:0;229bab1f9d30:45245 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T21:02:11,605 INFO [M:0;229bab1f9d30:45245 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T21:02:11,606 INFO [M:0;229bab1f9d30:45245 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T21:02:11,606 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T21:02:11,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T21:02:11,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:11,612 DEBUG [M:0;229bab1f9d30:45245 {}] zookeeper.ZKUtil(347): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T21:02:11,612 WARN [M:0;229bab1f9d30:45245 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T21:02:11,613 INFO [M:0;229bab1f9d30:45245 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/.lastflushedseqids 2024-12-08T21:02:11,622 INFO [M:0;229bab1f9d30:45245 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T21:02:11,622 INFO [M:0;229bab1f9d30:45245 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T21:02:11,622 DEBUG [M:0;229bab1f9d30:45245 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T21:02:11,622 INFO [M:0;229bab1f9d30:45245 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:11,622 DEBUG [M:0;229bab1f9d30:45245 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:11,622 DEBUG [M:0;229bab1f9d30:45245 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T21:02:11,622 DEBUG [M:0;229bab1f9d30:45245 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:11,623 INFO [M:0;229bab1f9d30:45245 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-12-08T21:02:11,631 DEBUG [M:0;229bab1f9d30:45245 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8ff20e173138449eac9b79fe8826a31c 2024-12-08T21:02:11,645 DEBUG [M:0;229bab1f9d30:45245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8ff20e173138449eac9b79fe8826a31c is 82, key is hbase:meta,,1/info:regioninfo/1733691728365/Put/seqid=0 2024-12-08T21:02:11,645 INFO [M:0;229bab1f9d30:45245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8ff20e173138449eac9b79fe8826a31c 2024-12-08T21:02:11,651 DEBUG [M:0;229bab1f9d30:45245 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4ea9802e835c42868bd6a15a7b696188 2024-12-08T21:02:11,661 DEBUG [M:0;229bab1f9d30:45245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ea9802e835c42868bd6a15a7b696188 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733691728853/Put/seqid=0 2024-12-08T21:02:11,662 INFO [M:0;229bab1f9d30:45245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ea9802e835c42868bd6a15a7b696188 2024-12-08T21:02:11,668 DEBUG [M:0;229bab1f9d30:45245 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 00b45d44c13943fc9da17e2726e28481 2024-12-08T21:02:11,679 DEBUG [M:0;229bab1f9d30:45245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/00b45d44c13943fc9da17e2726e28481 is 69, key is 229bab1f9d30,33091,1733691727294/rs:state/1733691727805/Put/seqid=0 2024-12-08T21:02:11,680 INFO [M:0;229bab1f9d30:45245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/00b45d44c13943fc9da17e2726e28481 2024-12-08T21:02:11,681 INFO [M:0;229bab1f9d30:45245 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00b45d44c13943fc9da17e2726e28481 2024-12-08T21:02:11,682 DEBUG [M:0;229bab1f9d30:45245 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8ff20e173138449eac9b79fe8826a31c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8ff20e173138449eac9b79fe8826a31c 2024-12-08T21:02:11,683 INFO [M:0;229bab1f9d30:45245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8ff20e173138449eac9b79fe8826a31c, entries=8, sequenceid=83, filesize=5.5 K 2024-12-08T21:02:11,684 DEBUG [M:0;229bab1f9d30:45245 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ea9802e835c42868bd6a15a7b696188 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ea9802e835c42868bd6a15a7b696188 2024-12-08T21:02:11,685 INFO [M:0;229bab1f9d30:45245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ea9802e835c42868bd6a15a7b696188, entries=9, sequenceid=83, filesize=6.4 K 2024-12-08T21:02:11,685 DEBUG [M:0;229bab1f9d30:45245 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/00b45d44c13943fc9da17e2726e28481 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/00b45d44c13943fc9da17e2726e28481 2024-12-08T21:02:11,686 INFO [M:0;229bab1f9d30:45245 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00b45d44c13943fc9da17e2726e28481 2024-12-08T21:02:11,687 INFO [M:0;229bab1f9d30:45245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/560b821f-71cd-20b4-4f63-40e534f4d46e/c33567f8-64a8-b04c-79f5-fdd0053a6200/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/00b45d44c13943fc9da17e2726e28481, entries=3, sequenceid=83, filesize=5.3 K 2024-12-08T21:02:11,687 INFO [M:0;229bab1f9d30:45245 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 65ms, sequenceid=83, compaction requested=false 2024-12-08T21:02:11,689 INFO [M:0;229bab1f9d30:45245 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:11,689 DEBUG [M:0;229bab1f9d30:45245 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733691731622Disabling compacts and flushes for region at 1733691731622Disabling writes for close at 1733691731622Obtaining lock to block concurrent updates at 1733691731623 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733691731623Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1733691731623Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733691731623Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733691731623Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733691731644 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733691731644Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733691731646 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733691731661 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733691731661Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733691731663 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733691731678 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733691731678Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@739df619: reopening flushed file at 1733691731681 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@87d4814: reopening flushed file at 1733691731683 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a29d9ca: reopening flushed file at 1733691731685 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 65ms, sequenceid=83, compaction requested=false at 1733691731687 (+2 ms)Writing region close event to WAL at 1733691731689 (+2 ms)Closed at 1733691731689 2024-12-08T21:02:11,691 INFO [M:0;229bab1f9d30:45245 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T21:02:11,691 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:02:11,691 INFO [M:0;229bab1f9d30:45245 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45245 2024-12-08T21:02:11,691 INFO [M:0;229bab1f9d30:45245 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:02:11,695 INFO [RS:0;229bab1f9d30:33091 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:02:11,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:11,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x100074a1bb60001, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:11,695 INFO [RS:0;229bab1f9d30:33091 {}] regionserver.HRegionServer(1031): Exiting; stopping=229bab1f9d30,33091,1733691727294; zookeeper connection closed. 2024-12-08T21:02:11,696 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@35c72c69 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@35c72c69 2024-12-08T21:02:11,696 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-08T21:02:11,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:11,812 INFO [M:0;229bab1f9d30:45245 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:02:11,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45245-0x100074a1bb60000, quorum=127.0.0.1:60212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:02:11,815 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T21:02:11,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-08T21:02:11,816 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:11,822 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 383), ProcessCount=11 (was 11), AvailableMemoryMB=16795 (was 16869)