2024-12-05 19:45:13,446 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-05 19:45:13,470 main DEBUG Took 0.021797 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 19:45:13,471 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 19:45:13,471 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 19:45:13,472 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 19:45:13,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,482 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 19:45:13,496 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,498 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,499 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,500 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,501 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,502 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,502 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,503 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,504 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,505 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,505 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,506 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,506 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,507 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,507 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,508 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,508 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,508 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,509 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,509 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,510 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 19:45:13,511 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,511 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 19:45:13,513 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 19:45:13,514 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 19:45:13,516 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 19:45:13,517 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 19:45:13,518 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 19:45:13,519 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 19:45:13,530 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 19:45:13,532 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 19:45:13,534 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 19:45:13,534 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 19:45:13,535 main DEBUG createAppenders(={Console}) 2024-12-05 19:45:13,536 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-05 19:45:13,536 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-05 19:45:13,536 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-05 19:45:13,537 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 19:45:13,538 main DEBUG OutputStream closed 2024-12-05 19:45:13,538 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 19:45:13,538 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 19:45:13,539 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-05 19:45:13,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 19:45:13,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 19:45:13,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 19:45:13,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 19:45:13,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 19:45:13,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 19:45:13,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 19:45:13,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 19:45:13,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 19:45:13,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 19:45:13,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 19:45:13,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 19:45:13,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 19:45:13,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 19:45:13,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 19:45:13,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 19:45:13,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 19:45:13,632 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 19:45:13,635 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 19:45:13,635 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-05 19:45:13,635 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 19:45:13,636 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-05T19:45:13,656 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-05 19:45:13,660 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 19:45:13,661 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T19:45:13,914 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e 2024-12-05T19:45:13,939 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=743, ProcessCount=11, AvailableMemoryMB=9361 2024-12-05T19:45:14,047 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T19:45:14,152 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:14,167 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/cluster_f6382b24-6ade-90f2-7bdf-4e640f8757f2, deleteOnExit=true 2024-12-05T19:45:14,271 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/cluster_f6382b24-6ade-90f2-7bdf-4e640f8757f2/zookeeper_0, clientPort=59462, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/cluster_f6382b24-6ade-90f2-7bdf-4e640f8757f2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/cluster_f6382b24-6ade-90f2-7bdf-4e640f8757f2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T19:45:14,286 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59462 2024-12-05T19:45:14,287 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:14,403 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 with version=8 2024-12-05T19:45:14,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/hbase-staging 2024-12-05T19:45:14,512 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T19:45:14,785 INFO [Time-limited test {}] client.ConnectionUtils(128): master/86162e2766a8:0 server-side Connection retries=45 2024-12-05T19:45:14,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:14,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:14,807 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T19:45:14,807 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:14,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T19:45:14,959 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T19:45:15,060 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T19:45:15,102 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T19:45:15,113 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T19:45:15,117 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T19:45:15,163 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14588 (auto-detected) 2024-12-05T19:45:15,165 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T19:45:15,189 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33193 2024-12-05T19:45:15,216 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33193 connecting to ZooKeeper ensemble=127.0.0.1:59462 2024-12-05T19:45:15,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:331930x0, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T19:45:15,255 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33193-0x10063b69eeb0000 connected 2024-12-05T19:45:15,285 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:15,286 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:15,293 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:15,298 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7, hbase.cluster.distributed=false 2024-12-05T19:45:15,324 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T19:45:15,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33193 2024-12-05T19:45:15,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33193 2024-12-05T19:45:15,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33193 2024-12-05T19:45:15,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33193 2024-12-05T19:45:15,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33193 2024-12-05T19:45:15,423 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/86162e2766a8:0 server-side Connection retries=45 2024-12-05T19:45:15,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:15,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:15,425 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T19:45:15,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:15,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T19:45:15,429 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T19:45:15,431 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T19:45:15,432 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41649 2024-12-05T19:45:15,434 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41649 connecting to ZooKeeper ensemble=127.0.0.1:59462 2024-12-05T19:45:15,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:15,436 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:15,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416490x0, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T19:45:15,441 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41649-0x10063b69eeb0001 connected 2024-12-05T19:45:15,441 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:15,449 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T19:45:15,465 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.cache 2024-12-05T19:45:15,473 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T19:45:15,474 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T19:45:15,476 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T19:45:15,478 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T19:45:15,479 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T19:45:15,486 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T19:45:15,490 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T19:45:15,491 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.persistence is complete. 2024-12-05T19:45:15,493 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T19:45:15,496 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T19:45:15,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T19:45:15,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41649 2024-12-05T19:45:15,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41649 2024-12-05T19:45:15,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41649 2024-12-05T19:45:15,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41649 2024-12-05T19:45:15,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41649 2024-12-05T19:45:15,542 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/86162e2766a8:0 server-side Connection retries=45 2024-12-05T19:45:15,542 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:15,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:15,543 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T19:45:15,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:15,544 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T19:45:15,544 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T19:45:15,544 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T19:45:15,548 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40199 2024-12-05T19:45:15,551 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40199 connecting to ZooKeeper ensemble=127.0.0.1:59462 2024-12-05T19:45:15,551 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:15,552 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:15,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401990x0, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T19:45:15,559 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40199-0x10063b69eeb0002 connected 2024-12-05T19:45:15,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:15,560 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T19:45:15,561 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.cache 2024-12-05T19:45:15,561 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T19:45:15,562 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T19:45:15,562 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T19:45:15,564 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T19:45:15,564 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T19:45:15,565 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T19:45:15,565 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T19:45:15,565 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T19:45:15,566 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.persistence is complete. 2024-12-05T19:45:15,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T19:45:15,569 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T19:45:15,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40199 2024-12-05T19:45:15,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40199 2024-12-05T19:45:15,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40199 2024-12-05T19:45:15,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40199 2024-12-05T19:45:15,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40199 2024-12-05T19:45:15,596 DEBUG [M:0;86162e2766a8:33193 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;86162e2766a8:33193 2024-12-05T19:45:15,600 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/86162e2766a8,33193,1733427914584 2024-12-05T19:45:15,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:15,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:15,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:15,611 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/86162e2766a8,33193,1733427914584 2024-12-05T19:45:15,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T19:45:15,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:15,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T19:45:15,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:15,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:15,637 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T19:45:15,638 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/86162e2766a8,33193,1733427914584 from backup master directory 2024-12-05T19:45:15,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:15,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/86162e2766a8,33193,1733427914584 2024-12-05T19:45:15,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:15,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:15,642 WARN [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T19:45:15,643 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=86162e2766a8,33193,1733427914584 2024-12-05T19:45:15,645 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T19:45:15,647 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T19:45:15,655 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/hbase.id] with ID: 10c4c6cd-8834-4d25-a758-08fbaa48e0a2 2024-12-05T19:45:15,655 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/.tmp/hbase.id 2024-12-05T19:45:15,662 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/hbase.id] 2024-12-05T19:45:15,707 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:15,714 INFO [master/86162e2766a8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T19:45:15,736 INFO [master/86162e2766a8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-12-05T19:45:15,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:15,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:15,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:15,774 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T19:45:15,777 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T19:45:15,826 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T19:45:15,832 INFO [master/86162e2766a8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T19:45:15,894 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store 2024-12-05T19:45:15,921 INFO [master/86162e2766a8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T19:45:15,925 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:15,927 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T19:45:15,927 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:15,928 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:15,930 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T19:45:15,930 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:15,930 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:15,931 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733427915927Disabling compacts and flushes for region at 1733427915927Disabling writes for close at 1733427915930 (+3 ms)Writing region close event to WAL at 1733427915930Closed at 1733427915930 2024-12-05T19:45:15,932 WARN [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/.initializing 2024-12-05T19:45:15,933 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/WALs/86162e2766a8,33193,1733427914584 2024-12-05T19:45:15,949 INFO [master/86162e2766a8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T19:45:15,967 INFO [master/86162e2766a8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=86162e2766a8%2C33193%2C1733427914584, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/WALs/86162e2766a8,33193,1733427914584, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/oldWALs, maxLogs=10 2024-12-05T19:45:16,020 INFO [master/86162e2766a8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/WALs/86162e2766a8,33193,1733427914584/86162e2766a8%2C33193%2C1733427914584.1733427915974 2024-12-05T19:45:16,021 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T19:45:16,022 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T19:45:16,023 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:16,026 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,028 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,076 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T19:45:16,120 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:16,123 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:16,123 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T19:45:16,129 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:16,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:16,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T19:45:16,135 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:16,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:16,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T19:45:16,140 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:16,141 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:16,141 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,144 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,145 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,148 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,148 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,151 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T19:45:16,153 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:16,160 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T19:45:16,161 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74455659, jitterRate=0.10947577655315399}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T19:45:16,168 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733427916042Initializing all the Stores at 1733427916043 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427916043Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427916044 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427916045 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427916045Cleaning up temporary data from old regions at 1733427916149 (+104 ms)Region opened successfully at 1733427916168 (+19 ms) 2024-12-05T19:45:16,170 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T19:45:16,203 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226a2b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=86162e2766a8/172.17.0.2:0 2024-12-05T19:45:16,237 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T19:45:16,249 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T19:45:16,249 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T19:45:16,252 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T19:45:16,253 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T19:45:16,259 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T19:45:16,259 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T19:45:16,285 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T19:45:16,295 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T19:45:16,297 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T19:45:16,300 INFO [master/86162e2766a8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T19:45:16,302 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T19:45:16,304 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T19:45:16,306 INFO [master/86162e2766a8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T19:45:16,310 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T19:45:16,312 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T19:45:16,314 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T19:45:16,316 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T19:45:16,341 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T19:45:16,343 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T19:45:16,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:16,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:16,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:16,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:16,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:16,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:16,363 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=86162e2766a8,33193,1733427914584, sessionid=0x10063b69eeb0000, setting cluster-up flag (Was=false) 2024-12-05T19:45:16,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:16,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:16,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:16,385 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T19:45:16,387 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=86162e2766a8,33193,1733427914584 2024-12-05T19:45:16,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:16,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:16,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:16,399 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T19:45:16,401 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=86162e2766a8,33193,1733427914584 2024-12-05T19:45:16,407 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T19:45:16,481 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T19:45:16,483 INFO [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(746): ClusterId : 10c4c6cd-8834-4d25-a758-08fbaa48e0a2 2024-12-05T19:45:16,484 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(746): ClusterId : 10c4c6cd-8834-4d25-a758-08fbaa48e0a2 2024-12-05T19:45:16,486 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T19:45:16,486 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T19:45:16,493 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T19:45:16,493 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T19:45:16,493 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T19:45:16,493 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T19:45:16,494 INFO [master/86162e2766a8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T19:45:16,497 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T19:45:16,497 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T19:45:16,498 DEBUG [RS:0;86162e2766a8:41649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6681f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=86162e2766a8/172.17.0.2:0 2024-12-05T19:45:16,498 DEBUG [RS:1;86162e2766a8:40199 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=86162e2766a8/172.17.0.2:0 2024-12-05T19:45:16,501 INFO [master/86162e2766a8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T19:45:16,509 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 86162e2766a8,33193,1733427914584 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T19:45:16,514 DEBUG [RS:0;86162e2766a8:41649 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;86162e2766a8:41649 2024-12-05T19:45:16,514 DEBUG [RS:1;86162e2766a8:40199 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;86162e2766a8:40199 2024-12-05T19:45:16,517 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T19:45:16,517 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T19:45:16,517 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T19:45:16,517 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T19:45:16,517 INFO [RS:0;86162e2766a8:41649 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T19:45:16,517 INFO [RS:1;86162e2766a8:40199 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T19:45:16,517 INFO [RS:0;86162e2766a8:41649 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T19:45:16,517 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/86162e2766a8:0, corePoolSize=10, maxPoolSize=10 2024-12-05T19:45:16,517 INFO [RS:1;86162e2766a8:40199 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T19:45:16,517 DEBUG [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T19:45:16,518 DEBUG [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T19:45:16,518 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,518 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/86162e2766a8:0, corePoolSize=2, maxPoolSize=2 2024-12-05T19:45:16,518 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,521 INFO [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(2659): reportForDuty to master=86162e2766a8,33193,1733427914584 with port=40199, startcode=1733427915541 2024-12-05T19:45:16,520 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(2659): reportForDuty to master=86162e2766a8,33193,1733427914584 with port=41649, startcode=1733427915382 2024-12-05T19:45:16,521 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733427946521 2024-12-05T19:45:16,523 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T19:45:16,524 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T19:45:16,524 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T19:45:16,525 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T19:45:16,528 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T19:45:16,529 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T19:45:16,529 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T19:45:16,529 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T19:45:16,530 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:16,531 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T19:45:16,533 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,537 DEBUG [RS:0;86162e2766a8:41649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T19:45:16,537 DEBUG [RS:1;86162e2766a8:40199 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T19:45:16,544 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T19:45:16,546 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T19:45:16,546 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T19:45:16,552 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T19:45:16,552 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T19:45:16,554 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.large.0-1733427916553,5,FailOnTimeoutGroup] 2024-12-05T19:45:16,556 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.small.0-1733427916554,5,FailOnTimeoutGroup] 2024-12-05T19:45:16,556 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,556 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T19:45:16,558 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,558 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,563 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T19:45:16,564 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 2024-12-05T19:45:16,581 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:16,582 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T19:45:16,597 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T19:45:16,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:16,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:16,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T19:45:16,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T19:45:16,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:16,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:16,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T19:45:16,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T19:45:16,622 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:16,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:16,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T19:45:16,624 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37527, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T19:45:16,624 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33111, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T19:45:16,632 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33193 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 86162e2766a8,40199,1733427915541 2024-12-05T19:45:16,636 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33193 {}] master.ServerManager(517): Registering regionserver=86162e2766a8,40199,1733427915541 2024-12-05T19:45:16,638 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T19:45:16,638 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:16,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:16,639 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T19:45:16,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740 2024-12-05T19:45:16,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740 2024-12-05T19:45:16,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T19:45:16,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T19:45:16,641 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T19:45:16,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T19:45:16,650 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33193 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 86162e2766a8,41649,1733427915382 2024-12-05T19:45:16,651 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33193 {}] master.ServerManager(517): Registering regionserver=86162e2766a8,41649,1733427915382 2024-12-05T19:45:16,654 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T19:45:16,655 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65271849, jitterRate=-0.027373656630516052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T19:45:16,656 DEBUG [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 2024-12-05T19:45:16,656 DEBUG [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 2024-12-05T19:45:16,656 DEBUG [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T19:45:16,656 DEBUG [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T19:45:16,656 DEBUG [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T19:45:16,656 DEBUG [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T19:45:16,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733427916581Initializing all the Stores at 1733427916582 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427916582Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427916582Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427916582Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427916582Cleaning up temporary data from old regions at 1733427916641 (+59 ms)Region opened successfully at 1733427916659 (+18 ms) 2024-12-05T19:45:16,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T19:45:16,660 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T19:45:16,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T19:45:16,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T19:45:16,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T19:45:16,662 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T19:45:16,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733427916659Disabling compacts and flushes for region at 1733427916659Disabling writes for close at 1733427916660 (+1 ms)Writing region close event to WAL at 1733427916661 (+1 ms)Closed at 1733427916662 (+1 ms) 2024-12-05T19:45:16,664 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T19:45:16,664 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T19:45:16,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T19:45:16,665 DEBUG [RS:0;86162e2766a8:41649 {}] zookeeper.ZKUtil(111): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/86162e2766a8,41649,1733427915382 2024-12-05T19:45:16,666 WARN [RS:0;86162e2766a8:41649 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T19:45:16,666 INFO [RS:0;86162e2766a8:41649 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T19:45:16,666 DEBUG [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,41649,1733427915382 2024-12-05T19:45:16,668 DEBUG [RS:1;86162e2766a8:40199 {}] zookeeper.ZKUtil(111): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/86162e2766a8,40199,1733427915541 2024-12-05T19:45:16,668 WARN [RS:1;86162e2766a8:40199 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T19:45:16,668 INFO [RS:1;86162e2766a8:40199 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T19:45:16,668 DEBUG [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,40199,1733427915541 2024-12-05T19:45:16,670 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [86162e2766a8,40199,1733427915541] 2024-12-05T19:45:16,670 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [86162e2766a8,41649,1733427915382] 2024-12-05T19:45:16,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T19:45:16,684 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T19:45:16,687 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T19:45:16,698 INFO [RS:1;86162e2766a8:40199 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T19:45:16,698 INFO [RS:0;86162e2766a8:41649 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T19:45:16,715 INFO [RS:1;86162e2766a8:40199 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T19:45:16,715 INFO [RS:0;86162e2766a8:41649 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T19:45:16,721 INFO [RS:0;86162e2766a8:41649 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T19:45:16,721 INFO [RS:1;86162e2766a8:40199 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T19:45:16,722 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,723 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,727 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T19:45:16,727 INFO [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T19:45:16,733 INFO [RS:0;86162e2766a8:41649 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T19:45:16,733 INFO [RS:1;86162e2766a8:40199 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T19:45:16,735 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,735 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,735 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,736 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,736 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,736 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,736 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,736 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,736 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,736 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,736 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,736 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/86162e2766a8:0, corePoolSize=2, maxPoolSize=2 2024-12-05T19:45:16,736 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/86162e2766a8:0, corePoolSize=2, maxPoolSize=2 2024-12-05T19:45:16,737 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,737 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,738 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,738 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:16,738 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:16,738 DEBUG [RS:0;86162e2766a8:41649 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:16,738 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:16,738 DEBUG [RS:1;86162e2766a8:40199 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:16,744 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,744 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,744 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,744 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,745 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,745 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,745 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,745 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,745 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,40199,1733427915541-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T19:45:16,745 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,745 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,746 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,41649,1733427915382-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T19:45:16,774 INFO [RS:1;86162e2766a8:40199 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T19:45:16,774 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T19:45:16,777 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,40199,1733427915541-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,777 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,41649,1733427915382-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,777 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,777 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,777 INFO [RS:0;86162e2766a8:41649 {}] regionserver.Replication(171): 86162e2766a8,41649,1733427915382 started 2024-12-05T19:45:16,777 INFO [RS:1;86162e2766a8:40199 {}] regionserver.Replication(171): 86162e2766a8,40199,1733427915541 started 2024-12-05T19:45:16,800 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,801 INFO [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(1482): Serving as 86162e2766a8,40199,1733427915541, RpcServer on 86162e2766a8/172.17.0.2:40199, sessionid=0x10063b69eeb0002 2024-12-05T19:45:16,802 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T19:45:16,802 DEBUG [RS:1;86162e2766a8:40199 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 86162e2766a8,40199,1733427915541 2024-12-05T19:45:16,802 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,40199,1733427915541' 2024-12-05T19:45:16,802 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T19:45:16,804 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T19:45:16,805 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T19:45:16,805 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T19:45:16,805 DEBUG [RS:1;86162e2766a8:40199 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 86162e2766a8,40199,1733427915541 2024-12-05T19:45:16,805 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,40199,1733427915541' 2024-12-05T19:45:16,805 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T19:45:16,805 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:16,806 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(1482): Serving as 86162e2766a8,41649,1733427915382, RpcServer on 86162e2766a8/172.17.0.2:41649, sessionid=0x10063b69eeb0001 2024-12-05T19:45:16,806 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T19:45:16,806 DEBUG [RS:0;86162e2766a8:41649 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 86162e2766a8,41649,1733427915382 2024-12-05T19:45:16,806 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,41649,1733427915382' 2024-12-05T19:45:16,806 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T19:45:16,806 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T19:45:16,807 DEBUG [RS:1;86162e2766a8:40199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T19:45:16,807 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T19:45:16,807 INFO [RS:1;86162e2766a8:40199 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T19:45:16,808 INFO [RS:1;86162e2766a8:40199 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T19:45:16,808 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T19:45:16,808 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T19:45:16,808 DEBUG [RS:0;86162e2766a8:41649 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 86162e2766a8,41649,1733427915382 2024-12-05T19:45:16,808 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,41649,1733427915382' 2024-12-05T19:45:16,808 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T19:45:16,809 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T19:45:16,810 DEBUG [RS:0;86162e2766a8:41649 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T19:45:16,810 INFO [RS:0;86162e2766a8:41649 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T19:45:16,810 INFO [RS:0;86162e2766a8:41649 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T19:45:16,838 WARN [86162e2766a8:33193 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T19:45:16,914 INFO [RS:0;86162e2766a8:41649 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T19:45:16,914 INFO [RS:1;86162e2766a8:40199 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T19:45:16,915 INFO [RS:1;86162e2766a8:40199 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=86162e2766a8%2C40199%2C1733427915541, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,40199,1733427915541, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs, maxLogs=55 2024-12-05T19:45:16,915 INFO [RS:0;86162e2766a8:41649 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=86162e2766a8%2C41649%2C1733427915382, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,41649,1733427915382, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs, maxLogs=55 2024-12-05T19:45:16,942 INFO [RS:0;86162e2766a8:41649 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,41649,1733427915382/86162e2766a8%2C41649%2C1733427915382.1733427916921 2024-12-05T19:45:16,942 INFO [RS:1;86162e2766a8:40199 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,40199,1733427915541/86162e2766a8%2C40199%2C1733427915541.1733427916920 2024-12-05T19:45:16,943 DEBUG [RS:0;86162e2766a8:41649 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T19:45:16,943 DEBUG [RS:1;86162e2766a8:40199 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T19:45:17,091 DEBUG [86162e2766a8:33193 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-05T19:45:17,103 DEBUG [86162e2766a8:33193 {}] balancer.BalancerClusterState(204): Hosts are {86162e2766a8=0} racks are {/default-rack=0} 2024-12-05T19:45:17,106 DEBUG [86162e2766a8:33193 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T19:45:17,106 DEBUG [86162e2766a8:33193 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T19:45:17,106 DEBUG [86162e2766a8:33193 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T19:45:17,106 DEBUG [86162e2766a8:33193 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T19:45:17,107 INFO [86162e2766a8:33193 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T19:45:17,107 INFO [86162e2766a8:33193 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T19:45:17,107 DEBUG [86162e2766a8:33193 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T19:45:17,114 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=86162e2766a8,41649,1733427915382 2024-12-05T19:45:17,119 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 86162e2766a8,41649,1733427915382, state=OPENING 2024-12-05T19:45:17,125 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T19:45:17,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:17,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:17,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:17,128 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:17,128 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:17,128 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:17,130 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T19:45:17,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=86162e2766a8,41649,1733427915382}] 2024-12-05T19:45:17,306 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T19:45:17,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42113, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T19:45:17,319 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T19:45:17,319 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T19:45:17,320 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T19:45:17,321 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=86162e2766a8%2C41649%2C1733427915382.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,41649,1733427915382, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs, maxLogs=55 2024-12-05T19:45:17,342 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,41649,1733427915382/86162e2766a8%2C41649%2C1733427915382.meta.1733427917323.meta 2024-12-05T19:45:17,343 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T19:45:17,343 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T19:45:17,345 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T19:45:17,348 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T19:45:17,352 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T19:45:17,356 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T19:45:17,357 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:17,357 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T19:45:17,357 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T19:45:17,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T19:45:17,360 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T19:45:17,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:17,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:17,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T19:45:17,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T19:45:17,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:17,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:17,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T19:45:17,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T19:45:17,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:17,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:17,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T19:45:17,365 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T19:45:17,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:17,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:17,366 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T19:45:17,367 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740 2024-12-05T19:45:17,368 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740 2024-12-05T19:45:17,368 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T19:45:17,368 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T19:45:17,369 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T19:45:17,370 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T19:45:17,370 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62996707, jitterRate=-0.061275914311409}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T19:45:17,371 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T19:45:17,373 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733427917358Writing region info on filesystem at 1733427917358Initializing all the Stores at 1733427917358Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427917358Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427917358Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427917359 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427917359Cleaning up temporary data from old regions at 1733427917369 (+10 ms)Running coprocessor post-open hooks at 1733427917371 (+2 ms)Region opened successfully at 1733427917373 (+2 ms) 2024-12-05T19:45:17,379 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733427917297 2024-12-05T19:45:17,390 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T19:45:17,391 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T19:45:17,392 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,41649,1733427915382 2024-12-05T19:45:17,393 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 86162e2766a8,41649,1733427915382, state=OPEN 2024-12-05T19:45:17,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T19:45:17,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T19:45:17,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T19:45:17,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:17,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:17,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:17,397 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=86162e2766a8,41649,1733427915382 2024-12-05T19:45:17,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T19:45:17,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=86162e2766a8,41649,1733427915382 in 266 msec 2024-12-05T19:45:17,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T19:45:17,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 730 msec 2024-12-05T19:45:17,407 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T19:45:17,407 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T19:45:17,428 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T19:45:17,430 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=86162e2766a8,41649,1733427915382, seqNum=-1] 2024-12-05T19:45:17,450 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T19:45:17,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36421, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T19:45:17,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0340 sec 2024-12-05T19:45:17,472 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733427917471, completionTime=-1 2024-12-05T19:45:17,475 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-05T19:45:17,475 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T19:45:17,502 INFO [master/86162e2766a8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T19:45:17,502 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733427977502 2024-12-05T19:45:17,502 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733428037502 2024-12-05T19:45:17,502 INFO [master/86162e2766a8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-05T19:45:17,504 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T19:45:17,510 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,33193,1733427914584-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:17,511 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,33193,1733427914584-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:17,511 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,33193,1733427914584-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:17,512 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-86162e2766a8:33193, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:17,513 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:17,513 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:17,521 DEBUG [master/86162e2766a8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T19:45:17,542 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.899sec 2024-12-05T19:45:17,543 INFO [master/86162e2766a8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T19:45:17,545 INFO [master/86162e2766a8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T19:45:17,546 INFO [master/86162e2766a8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T19:45:17,546 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T19:45:17,546 INFO [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T19:45:17,547 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,33193,1733427914584-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T19:45:17,548 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,33193,1733427914584-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T19:45:17,549 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T19:45:17,550 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T19:45:17,550 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,33193,1733427914584-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:17,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8eba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T19:45:17,600 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T19:45:17,600 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T19:45:17,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 86162e2766a8,33193,-1 for getting cluster id 2024-12-05T19:45:17,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T19:45:17,618 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10c4c6cd-8834-4d25-a758-08fbaa48e0a2' 2024-12-05T19:45:17,621 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T19:45:17,621 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10c4c6cd-8834-4d25-a758-08fbaa48e0a2" 2024-12-05T19:45:17,623 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2314353, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T19:45:17,623 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [86162e2766a8,33193,-1] 2024-12-05T19:45:17,626 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T19:45:17,628 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:17,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T19:45:17,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T19:45:17,632 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T19:45:17,640 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=86162e2766a8,41649,1733427915382, seqNum=-1] 2024-12-05T19:45:17,640 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T19:45:17,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T19:45:17,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=86162e2766a8,33193,1733427914584 2024-12-05T19:45:17,667 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T19:45:17,684 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 86162e2766a8,33193,1733427914584 2024-12-05T19:45:17,687 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@88c4b7b 2024-12-05T19:45:17,689 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T19:45:17,694 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T19:45:17,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733427917663', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T19:45:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733427917663 2024-12-05T19:45:17,715 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733427917663 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T19:45:17,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733427917663" procId is: 4 2024-12-05T19:45:17,719 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:17,721 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733427917663 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T19:45:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T19:45:17,736 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733427917663-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2a9e5cecc5c7264c71c063f6ac8ac475, NAME => 'testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733427917663', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 2024-12-05T19:45:17,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733427917663-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:17,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733427917663-pool-0 {}] regionserver.HRegion(1722): Closing 2a9e5cecc5c7264c71c063f6ac8ac475, disabling compactions & flushes 2024-12-05T19:45:17,743 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733427917663-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:17,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733427917663-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:17,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733427917663-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. after waiting 0 ms 2024-12-05T19:45:17,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733427917663-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:17,743 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733427917663-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:17,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733427917663-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2a9e5cecc5c7264c71c063f6ac8ac475: Waiting for close lock at 1733427917743Disabling compacts and flushes for region at 1733427917743Disabling writes for close at 1733427917743Writing region close event to WAL at 1733427917743Closed at 1733427917743 2024-12-05T19:45:17,745 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733427917663 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T19:45:17,751 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733427917745"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733427917745"}]},"ts":"1733427917745"} 2024-12-05T19:45:17,757 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T19:45:17,758 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733427917663 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T19:45:17,762 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733427917663","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733427917758"}]},"ts":"1733427917758"} 2024-12-05T19:45:17,766 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733427917663, state=ENABLING in hbase:meta 2024-12-05T19:45:17,767 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(204): Hosts are {86162e2766a8=0} racks are {/default-rack=0} 2024-12-05T19:45:17,768 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T19:45:17,768 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T19:45:17,768 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T19:45:17,768 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T19:45:17,768 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T19:45:17,768 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T19:45:17,768 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T19:45:17,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, ASSIGN}] 2024-12-05T19:45:17,771 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, ASSIGN 2024-12-05T19:45:17,773 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, ASSIGN; state=OFFLINE, location=86162e2766a8,40199,1733427915541; forceNewPlan=false, retain=false 2024-12-05T19:45:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T19:45:17,925 INFO [86162e2766a8:33193 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T19:45:17,926 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2a9e5cecc5c7264c71c063f6ac8ac475, regionState=OPENING, regionLocation=86162e2766a8,40199,1733427915541 2024-12-05T19:45:17,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, ASSIGN because future has completed 2024-12-05T19:45:17,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a9e5cecc5c7264c71c063f6ac8ac475, server=86162e2766a8,40199,1733427915541}] 2024-12-05T19:45:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T19:45:18,084 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T19:45:18,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55919, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T19:45:18,089 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:18,089 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a9e5cecc5c7264c71c063f6ac8ac475, NAME => 'testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475.', STARTKEY => '', ENDKEY => ''} 2024-12-05T19:45:18,090 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733427917663 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,090 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:18,090 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,090 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,091 INFO [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,096 INFO [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a9e5cecc5c7264c71c063f6ac8ac475 columnFamilyName family 2024-12-05T19:45:18,097 DEBUG [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:18,097 INFO [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] regionserver.HStore(327): Store=2a9e5cecc5c7264c71c063f6ac8ac475/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:18,098 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,099 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,099 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,099 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,099 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,100 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,107 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T19:45:18,108 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2a9e5cecc5c7264c71c063f6ac8ac475; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59612759, jitterRate=-0.11170066893100739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T19:45:18,108 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:18,109 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2a9e5cecc5c7264c71c063f6ac8ac475: Running coprocessor pre-open hook at 1733427918090Writing region info on filesystem at 1733427918090Initializing all the Stores at 1733427918090Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427918090Cleaning up temporary data from old regions at 1733427918099 (+9 ms)Running coprocessor post-open hooks at 1733427918108 (+9 ms)Region opened successfully at 1733427918109 (+1 ms) 2024-12-05T19:45:18,110 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475., pid=6, masterSystemTime=1733427918083 2024-12-05T19:45:18,113 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:18,113 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:18,113 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2a9e5cecc5c7264c71c063f6ac8ac475, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,40199,1733427915541 2024-12-05T19:45:18,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a9e5cecc5c7264c71c063f6ac8ac475, server=86162e2766a8,40199,1733427915541 because future has completed 2024-12-05T19:45:18,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T19:45:18,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a9e5cecc5c7264c71c063f6ac8ac475, server=86162e2766a8,40199,1733427915541 in 187 msec 2024-12-05T19:45:18,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T19:45:18,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, ASSIGN in 351 msec 2024-12-05T19:45:18,124 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733427917663 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T19:45:18,124 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733427917663","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733427918124"}]},"ts":"1733427918124"} 2024-12-05T19:45:18,127 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733427917663, state=ENABLED in hbase:meta 2024-12-05T19:45:18,127 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733427917663 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T19:45:18,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733427917663 in 419 msec 2024-12-05T19:45:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T19:45:18,358 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733427917663 completed 2024-12-05T19:45:18,358 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733427917663 get assigned. Timeout = 60000ms 2024-12-05T19:45:18,361 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T19:45:18,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733427917663 assigned to meta. Checking AM states. 2024-12-05T19:45:18,369 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T19:45:18,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733427917663 assigned. 2024-12-05T19:45:18,379 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733427917663', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475., hostname=86162e2766a8,40199,1733427915541, seqNum=2] 2024-12-05T19:45:18,381 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T19:45:18,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T19:45:18,393 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2a9e5cecc5c7264c71c063f6ac8ac475 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-05T19:45:18,421 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d69f7d64847f4fc1a37f9edf52020162 2024-12-05T19:45:18,464 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/.tmp/family/d69f7d64847f4fc1a37f9edf52020162 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-05T19:45:18,469 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/.tmp/family/d69f7d64847f4fc1a37f9edf52020162 2024-12-05T19:45:18,505 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/.tmp/family/d69f7d64847f4fc1a37f9edf52020162 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162 2024-12-05T19:45:18,507 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162, delay=904 ms 2024-12-05T19:45:18,509 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T19:45:18,516 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 2a9e5cecc5c7264c71c063f6ac8ac475 in 121ms, sequenceid=6, compaction requested=false 2024-12-05T19:45:18,518 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733427917663' 2024-12-05T19:45:18,520 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2a9e5cecc5c7264c71c063f6ac8ac475: 2024-12-05T19:45:19,414 DEBUG [hfile-prefetch-1733427918507 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d69f7d64847f4fc1a37f9edf52020162_0. 2024-12-05T19:45:19,418 DEBUG [hfile-prefetch-1733427918507 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d69f7d64847f4fc1a37f9edf52020162_117. 2024-12-05T19:45:19,420 DEBUG [hfile-prefetch-1733427918507 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162, with total blocks 2, and data blocks 1 2024-12-05T19:45:19,420 DEBUG [hfile-prefetch-1733427918507 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-05T19:45:19,420 DEBUG [hfile-prefetch-1733427918507 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-05T19:45:19,421 DEBUG [hfile-prefetch-1733427918507 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162. Will sleep 100ms and try the verification again. 2024-12-05T19:45:19,467 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T19:45:19,467 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T19:45:19,468 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:19,468 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:19,482 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T19:45:19,482 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T19:45:19,482 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:19,482 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:19,486 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T19:45:19,486 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733427919482 2024-12-05T19:45:19,486 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T19:45:19,488 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-05T19:45:19,489 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733427919467 2024-12-05T19:45:19,489 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T19:45:19,521 DEBUG [hfile-prefetch-1733427918507 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162, with total blocks 2, and data blocks 1 2024-12-05T19:45:19,521 DEBUG [hfile-prefetch-1733427918507 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-05T19:45:19,521 DEBUG [hfile-prefetch-1733427918507 {}] bucket.BucketCache(2293): found block for file d69f7d64847f4fc1a37f9edf52020162 in the backing map. Acquiring read lock for offset 0 2024-12-05T19:45:19,521 DEBUG [hfile-prefetch-1733427918507 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162 has now been fully cached. 2024-12-05T19:45:19,522 DEBUG [hfile-prefetch-1733427918507 {}] hfile.PrefetchExecutor(121): Prefetch completed for d69f7d64847f4fc1a37f9edf52020162 2024-12-05T19:45:19,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=2a9e5cecc5c7264c71c063f6ac8ac475, source=86162e2766a8,40199,1733427915541, destination=86162e2766a8,41649,1733427915382, warming up region on 86162e2766a8,41649,1733427915382 2024-12-05T19:45:19,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=2a9e5cecc5c7264c71c063f6ac8ac475, source=86162e2766a8,40199,1733427915541, destination=86162e2766a8,41649,1733427915382, running balancer 2024-12-05T19:45:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, REOPEN/MOVE 2024-12-05T19:45:19,535 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, REOPEN/MOVE 2024-12-05T19:45:19,536 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.HRegion(7855): Warmup {ENCODED => 2a9e5cecc5c7264c71c063f6ac8ac475, NAME => 'testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475.', STARTKEY => '', ENDKEY => ''} 2024-12-05T19:45:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:19,537 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2a9e5cecc5c7264c71c063f6ac8ac475, regionState=CLOSING, regionLocation=86162e2766a8,40199,1733427915541 2024-12-05T19:45:19,537 INFO [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:19,538 INFO [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a9e5cecc5c7264c71c063f6ac8ac475 columnFamilyName family 2024-12-05T19:45:19,538 DEBUG [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:19,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, REOPEN/MOVE because future has completed 2024-12-05T19:45:19,540 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-05T19:45:19,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a9e5cecc5c7264c71c063f6ac8ac475, server=86162e2766a8,40199,1733427915541}] 2024-12-05T19:45:19,542 DEBUG [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162 2024-12-05T19:45:19,542 INFO [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] regionserver.HStore(327): Store=2a9e5cecc5c7264c71c063f6ac8ac475/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.HRegion(1722): Closing 2a9e5cecc5c7264c71c063f6ac8ac475, disabling compactions & flushes 2024-12-05T19:45:19,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. after waiting 0 ms 2024-12-05T19:45:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:19,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41649 {}] regionserver.HRegion(1676): Region close journal for 2a9e5cecc5c7264c71c063f6ac8ac475: Waiting for close lock at 1733427919544Disabling compacts and flushes for region at 1733427919544Disabling writes for close at 1733427919544Writing region close event to WAL at 1733427919546 (+2 ms)Closed at 1733427919546 2024-12-05T19:45:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33193 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-05T19:45:19,701 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:19,702 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-05T19:45:19,702 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 2a9e5cecc5c7264c71c063f6ac8ac475, disabling compactions & flushes 2024-12-05T19:45:19,703 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:19,703 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:19,703 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. after waiting 0 ms 2024-12-05T19:45:19,703 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:19,704 DEBUG [StoreFileCloser-2a9e5cecc5c7264c71c063f6ac8ac475-family-1 {}] bucket.BucketCache(768): Removing file d69f7d64847f4fc1a37f9edf52020162 for region 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:19,705 DEBUG [StoreFileCloser-2a9e5cecc5c7264c71c063f6ac8ac475-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file d69f7d64847f4fc1a37f9edf52020162, starting offset: 0, end offset: 9223372036854775807 2024-12-05T19:45:19,706 DEBUG [StoreFileCloser-2a9e5cecc5c7264c71c063f6ac8ac475-family-1 {}] bucket.BucketCache(856): removed key d69f7d64847f4fc1a37f9edf52020162_0 from back map with offset lock 2094080 in the evict process 2024-12-05T19:45:19,711 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T19:45:19,713 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:19,713 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 2a9e5cecc5c7264c71c063f6ac8ac475: Waiting for close lock at 1733427919702Running coprocessor pre-close hooks at 1733427919702Disabling compacts and flushes for region at 1733427919702Disabling writes for close at 1733427919703 (+1 ms)Writing region close event to WAL at 1733427919706 (+3 ms)Running coprocessor post-close hooks at 1733427919712 (+6 ms)Closed at 1733427919713 (+1 ms) 2024-12-05T19:45:19,713 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 2a9e5cecc5c7264c71c063f6ac8ac475 move to 86162e2766a8,41649,1733427915382 record at close sequenceid=6 2024-12-05T19:45:19,717 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:19,717 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2a9e5cecc5c7264c71c063f6ac8ac475, regionState=CLOSED 2024-12-05T19:45:19,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a9e5cecc5c7264c71c063f6ac8ac475, server=86162e2766a8,40199,1733427915541 because future has completed 2024-12-05T19:45:19,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T19:45:19,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 2a9e5cecc5c7264c71c063f6ac8ac475, server=86162e2766a8,40199,1733427915541 in 181 msec 2024-12-05T19:45:19,723 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, REOPEN/MOVE; state=CLOSED, location=86162e2766a8,41649,1733427915382; forceNewPlan=false, retain=false 2024-12-05T19:45:19,787 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T19:45:19,787 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T19:45:19,787 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:19,787 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:19,789 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T19:45:19,789 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T19:45:19,789 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:19,789 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:19,792 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T19:45:19,792 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733427919787 2024-12-05T19:45:19,792 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T19:45:19,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T19:45:19,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733427919789 2024-12-05T19:45:19,793 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T19:45:19,874 INFO [86162e2766a8:33193 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T19:45:19,874 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2a9e5cecc5c7264c71c063f6ac8ac475, regionState=OPENING, regionLocation=86162e2766a8,41649,1733427915382 2024-12-05T19:45:19,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, REOPEN/MOVE because future has completed 2024-12-05T19:45:19,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a9e5cecc5c7264c71c063f6ac8ac475, server=86162e2766a8,41649,1733427915382}] 2024-12-05T19:45:20,034 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:20,034 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a9e5cecc5c7264c71c063f6ac8ac475, NAME => 'testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475.', STARTKEY => '', ENDKEY => ''} 2024-12-05T19:45:20,035 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733427917663 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,035 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:20,035 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,035 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,036 INFO [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,037 INFO [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a9e5cecc5c7264c71c063f6ac8ac475 columnFamilyName family 2024-12-05T19:45:20,037 DEBUG [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:20,039 DEBUG [StoreFileOpener-2a9e5cecc5c7264c71c063f6ac8ac475-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162, delay=955 ms 2024-12-05T19:45:20,041 DEBUG [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162 2024-12-05T19:45:20,041 INFO [StoreOpener-2a9e5cecc5c7264c71c063f6ac8ac475-1 {}] regionserver.HStore(327): Store=2a9e5cecc5c7264c71c063f6ac8ac475/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:20,041 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,042 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,042 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,042 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,042 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,047 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,047 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 2a9e5cecc5c7264c71c063f6ac8ac475; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63494108, jitterRate=-0.05386406183242798}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T19:45:20,047 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,048 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 2a9e5cecc5c7264c71c063f6ac8ac475: Running coprocessor pre-open hook at 1733427920035Writing region info on filesystem at 1733427920035Initializing all the Stores at 1733427920036 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427920036Cleaning up temporary data from old regions at 1733427920042 (+6 ms)Running coprocessor post-open hooks at 1733427920048 (+6 ms)Region opened successfully at 1733427920048 2024-12-05T19:45:20,049 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475., pid=9, masterSystemTime=1733427920031 2024-12-05T19:45:20,052 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:20,052 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:20,053 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2a9e5cecc5c7264c71c063f6ac8ac475, regionState=OPEN, openSeqNum=10, regionLocation=86162e2766a8,41649,1733427915382 2024-12-05T19:45:20,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a9e5cecc5c7264c71c063f6ac8ac475, server=86162e2766a8,41649,1733427915382 because future has completed 2024-12-05T19:45:20,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-05T19:45:20,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a9e5cecc5c7264c71c063f6ac8ac475, server=86162e2766a8,41649,1733427915382 in 179 msec 2024-12-05T19:45:20,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733427917663, region=2a9e5cecc5c7264c71c063f6ac8ac475, REOPEN/MOVE in 525 msec 2024-12-05T19:45:20,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T19:45:20,088 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T19:45:20,088 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T19:45:20,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:20,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:20,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T19:45:20,094 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T19:45:20,095 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-12-05T19:45:20,095 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=86162e2766a8,33193,1733427914584 2024-12-05T19:45:20,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:20,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:20,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:20,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,098 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T19:45:20,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,099 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T19:45:20,099 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T19:45:20,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:20,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:20,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:20,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:20,100 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '86162e2766a8,41649,1733427915382' ***** 2024-12-05T19:45:20,100 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T19:45:20,100 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '86162e2766a8,40199,1733427915541' ***** 2024-12-05T19:45:20,100 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T19:45:20,100 INFO [RS:0;86162e2766a8:41649 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T19:45:20,100 INFO [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T19:45:20,101 INFO [RS:0;86162e2766a8:41649 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.persistence 2024-12-05T19:45:20,101 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T19:45:20,101 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T19:45:20,101 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T19:45:20,101 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T19:45:20,101 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T19:45:20,101 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T19:45:20,101 INFO [RS:0;86162e2766a8:41649 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T19:45:20,101 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T19:45:20,102 INFO [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/bucket.persistence 2024-12-05T19:45:20,102 INFO [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T19:45:20,102 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T19:45:20,111 DEBUG [RS:0;86162e2766a8:41649 {}] bucket.BucketCache(1487): Thread RS:0;86162e2766a8:41649 started persisting bucket cache to file 2024-12-05T19:45:20,112 DEBUG [RS:0;86162e2766a8:41649 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:20,112 DEBUG [RS:0;86162e2766a8:41649 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:20,112 DEBUG [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1487): Thread RS:1;86162e2766a8:40199 started persisting bucket cache to file 2024-12-05T19:45:20,112 DEBUG [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:20,112 DEBUG [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:20,116 DEBUG [RS:0;86162e2766a8:41649 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T19:45:20,116 DEBUG [RS:0;86162e2766a8:41649 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733427920112 2024-12-05T19:45:20,116 DEBUG [RS:0;86162e2766a8:41649 {}] bucket.BucketCache(1509): Thread RS:0;86162e2766a8:41649 finished persisting bucket cache to file, renaming 2024-12-05T19:45:20,116 WARN [Thread-97 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-12-05T19:45:20,116 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T19:45:20,116 ERROR [RS:1;86162e2766a8:40199 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T19:45:20,116 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T19:45:20,116 INFO [RS:0;86162e2766a8:41649 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T19:45:20,116 DEBUG [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T19:45:20,117 DEBUG [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733427920112 2024-12-05T19:45:20,117 DEBUG [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1509): Thread RS:1;86162e2766a8:40199 finished persisting bucket cache to file, renaming 2024-12-05T19:45:20,117 INFO [RS:0;86162e2766a8:41649 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T19:45:20,117 WARN [RS:1;86162e2766a8:40199 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-05T19:45:20,117 INFO [RS:1;86162e2766a8:40199 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T19:45:20,117 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(3091): Received CLOSE for 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,117 INFO [RS:1;86162e2766a8:40199 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T19:45:20,117 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T19:45:20,117 INFO [RS:1;86162e2766a8:40199 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T19:45:20,117 INFO [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(959): stopping server 86162e2766a8,40199,1733427915541 2024-12-05T19:45:20,117 INFO [RS:1;86162e2766a8:40199 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T19:45:20,117 INFO [RS:1;86162e2766a8:40199 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;86162e2766a8:40199. 2024-12-05T19:45:20,117 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(959): stopping server 86162e2766a8,41649,1733427915382 2024-12-05T19:45:20,118 DEBUG [RS:1;86162e2766a8:40199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T19:45:20,118 INFO [RS:0;86162e2766a8:41649 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T19:45:20,118 DEBUG [RS:1;86162e2766a8:40199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:20,118 INFO [RS:0;86162e2766a8:41649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;86162e2766a8:41649. 2024-12-05T19:45:20,118 DEBUG [RS:0;86162e2766a8:41649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T19:45:20,118 DEBUG [RS:0;86162e2766a8:41649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:20,118 INFO [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(976): stopping server 86162e2766a8,40199,1733427915541; all regions closed. 2024-12-05T19:45:20,118 INFO [RS:0;86162e2766a8:41649 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T19:45:20,118 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T19:45:20,118 INFO [RS:0;86162e2766a8:41649 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T19:45:20,118 INFO [RS:0;86162e2766a8:41649 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T19:45:20,118 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T19:45:20,119 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2a9e5cecc5c7264c71c063f6ac8ac475, disabling compactions & flushes 2024-12-05T19:45:20,119 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:20,119 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:20,119 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-05T19:45:20,119 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. after waiting 0 ms 2024-12-05T19:45:20,119 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T19:45:20,119 DEBUG [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(1325): Online Regions={2a9e5cecc5c7264c71c063f6ac8ac475=testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475., 1588230740=hbase:meta,,1.1588230740} 2024-12-05T19:45:20,119 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:20,119 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T19:45:20,119 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T19:45:20,119 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T19:45:20,120 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T19:45:20,120 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T19:45:20,120 DEBUG [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2a9e5cecc5c7264c71c063f6ac8ac475 2024-12-05T19:45:20,120 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-05T19:45:20,120 DEBUG [StoreFileCloser-2a9e5cecc5c7264c71c063f6ac8ac475-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/family/d69f7d64847f4fc1a37f9edf52020162 2024-12-05T19:45:20,124 DEBUG [RS:1;86162e2766a8:40199 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs 2024-12-05T19:45:20,124 INFO [RS:1;86162e2766a8:40199 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 86162e2766a8%2C40199%2C1733427915541:(num 1733427916920) 2024-12-05T19:45:20,124 DEBUG [RS:1;86162e2766a8:40199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:20,124 INFO [RS:1;86162e2766a8:40199 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:20,124 INFO [RS:1;86162e2766a8:40199 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T19:45:20,125 INFO [RS:1;86162e2766a8:40199 {}] hbase.ChoreService(370): Chore service for: regionserver/86162e2766a8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T19:45:20,125 INFO [RS:1;86162e2766a8:40199 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T19:45:20,125 INFO [RS:1;86162e2766a8:40199 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T19:45:20,125 INFO [RS:1;86162e2766a8:40199 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T19:45:20,125 INFO [regionserver/86162e2766a8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T19:45:20,126 INFO [RS:1;86162e2766a8:40199 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T19:45:20,126 INFO [RS:1;86162e2766a8:40199 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40199 2024-12-05T19:45:20,127 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnRegionMove1733427917663/2a9e5cecc5c7264c71c063f6ac8ac475/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-05T19:45:20,128 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:20,128 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2a9e5cecc5c7264c71c063f6ac8ac475: Waiting for close lock at 1733427920118Running coprocessor pre-close hooks at 1733427920119 (+1 ms)Disabling compacts and flushes for region at 1733427920119Disabling writes for close at 1733427920119Writing region close event to WAL at 1733427920121 (+2 ms)Running coprocessor post-close hooks at 1733427920128 (+7 ms)Closed at 1733427920128 2024-12-05T19:45:20,128 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475. 2024-12-05T19:45:20,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/86162e2766a8,40199,1733427915541 2024-12-05T19:45:20,132 INFO [RS:1;86162e2766a8:40199 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T19:45:20,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T19:45:20,135 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c917e35d43104024bb456ec8ab40a3e2 2024-12-05T19:45:20,135 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [86162e2766a8,40199,1733427915541] 2024-12-05T19:45:20,137 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/86162e2766a8,40199,1733427915541 already deleted, retry=false 2024-12-05T19:45:20,137 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 86162e2766a8,40199,1733427915541 expired; onlineServers=1 2024-12-05T19:45:20,150 INFO [regionserver/86162e2766a8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:20,150 INFO [regionserver/86162e2766a8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:20,152 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/info/c917e35d43104024bb456ec8ab40a3e2 is 205, key is testBlockEvictionOnRegionMove1733427917663,,1733427917696.2a9e5cecc5c7264c71c063f6ac8ac475./info:regioninfo/1733427920052/Put/seqid=0 2024-12-05T19:45:20,153 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/info/c917e35d43104024bb456ec8ab40a3e2 2024-12-05T19:45:20,163 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a494cc15673f4c7eb9df680397623be5 2024-12-05T19:45:20,177 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/ns/a494cc15673f4c7eb9df680397623be5 is 43, key is default/ns:d/1733427917457/Put/seqid=0 2024-12-05T19:45:20,178 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/ns/a494cc15673f4c7eb9df680397623be5 2024-12-05T19:45:20,186 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b84ddf7bf9b74d20b1897715d527f0ec 2024-12-05T19:45:20,200 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/table/b84ddf7bf9b74d20b1897715d527f0ec is 78, key is testBlockEvictionOnRegionMove1733427917663/table:state/1733427918124/Put/seqid=0 2024-12-05T19:45:20,201 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/table/b84ddf7bf9b74d20b1897715d527f0ec 2024-12-05T19:45:20,204 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/info/c917e35d43104024bb456ec8ab40a3e2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/info/c917e35d43104024bb456ec8ab40a3e2 2024-12-05T19:45:20,205 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/info/c917e35d43104024bb456ec8ab40a3e2, delay=929 ms 2024-12-05T19:45:20,206 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/info/c917e35d43104024bb456ec8ab40a3e2, entries=15, sequenceid=15, filesize=7.6 K 2024-12-05T19:45:20,207 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/ns/a494cc15673f4c7eb9df680397623be5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/ns/a494cc15673f4c7eb9df680397623be5 2024-12-05T19:45:20,208 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/ns/a494cc15673f4c7eb9df680397623be5, delay=965 ms 2024-12-05T19:45:20,210 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/ns/a494cc15673f4c7eb9df680397623be5, entries=2, sequenceid=15, filesize=5.0 K 2024-12-05T19:45:20,210 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/table/b84ddf7bf9b74d20b1897715d527f0ec as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/table/b84ddf7bf9b74d20b1897715d527f0ec 2024-12-05T19:45:20,211 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/table/b84ddf7bf9b74d20b1897715d527f0ec, delay=939 ms 2024-12-05T19:45:20,212 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/table/b84ddf7bf9b74d20b1897715d527f0ec, entries=2, sequenceid=15, filesize=5.3 K 2024-12-05T19:45:20,213 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false 2024-12-05T19:45:20,213 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T19:45:20,215 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/info/c917e35d43104024bb456ec8ab40a3e2 2024-12-05T19:45:20,216 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/ns/a494cc15673f4c7eb9df680397623be5 2024-12-05T19:45:20,216 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/table/b84ddf7bf9b74d20b1897715d527f0ec 2024-12-05T19:45:20,221 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-05T19:45:20,222 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T19:45:20,222 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T19:45:20,223 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733427920119Running coprocessor pre-close hooks at 1733427920119Disabling compacts and flushes for region at 1733427920119Disabling writes for close at 1733427920120 (+1 ms)Obtaining lock to block concurrent updates at 1733427920120Preparing flush snapshotting stores in 1588230740 at 1733427920120Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733427920121 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733427920121Flushing 1588230740/info: creating writer at 1733427920121Flushing 1588230740/info: appending metadata at 1733427920151 (+30 ms)Flushing 1588230740/info: closing flushed file at 1733427920151Flushing 1588230740/ns: creating writer at 1733427920156 (+5 ms)Flushing 1588230740/ns: appending metadata at 1733427920176 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733427920176Flushing 1588230740/table: creating writer at 1733427920180 (+4 ms)Flushing 1588230740/table: appending metadata at 1733427920200 (+20 ms)Flushing 1588230740/table: closing flushed file at 1733427920200Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42c13633: reopening flushed file at 1733427920203 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f3125a7: reopening flushed file at 1733427920207 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39389d6b: reopening flushed file at 1733427920210 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false at 1733427920213 (+3 ms)Writing region close event to WAL at 1733427920217 (+4 ms)Running coprocessor post-close hooks at 1733427920221 (+4 ms)Closed at 1733427920222 (+1 ms) 2024-12-05T19:45:20,223 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T19:45:20,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:20,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40199-0x10063b69eeb0002, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:20,235 INFO [RS:1;86162e2766a8:40199 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T19:45:20,236 INFO [RS:1;86162e2766a8:40199 {}] regionserver.HRegionServer(1031): Exiting; stopping=86162e2766a8,40199,1733427915541; zookeeper connection closed. 2024-12-05T19:45:20,236 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4673205e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4673205e 2024-12-05T19:45:20,320 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(976): stopping server 86162e2766a8,41649,1733427915382; all regions closed. 2024-12-05T19:45:20,323 DEBUG [RS:0;86162e2766a8:41649 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs 2024-12-05T19:45:20,323 INFO [RS:0;86162e2766a8:41649 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 86162e2766a8%2C41649%2C1733427915382.meta:.meta(num 1733427917323) 2024-12-05T19:45:20,325 DEBUG [RS:0;86162e2766a8:41649 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs 2024-12-05T19:45:20,325 INFO [RS:0;86162e2766a8:41649 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 86162e2766a8%2C41649%2C1733427915382:(num 1733427916921) 2024-12-05T19:45:20,325 DEBUG [RS:0;86162e2766a8:41649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:20,325 INFO [RS:0;86162e2766a8:41649 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:20,325 INFO [RS:0;86162e2766a8:41649 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T19:45:20,325 INFO [RS:0;86162e2766a8:41649 {}] hbase.ChoreService(370): Chore service for: regionserver/86162e2766a8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T19:45:20,326 INFO [RS:0;86162e2766a8:41649 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T19:45:20,326 INFO [regionserver/86162e2766a8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T19:45:20,326 INFO [RS:0;86162e2766a8:41649 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41649 2024-12-05T19:45:20,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T19:45:20,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/86162e2766a8,41649,1733427915382 2024-12-05T19:45:20,329 INFO [RS:0;86162e2766a8:41649 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T19:45:20,331 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [86162e2766a8,41649,1733427915382] 2024-12-05T19:45:20,332 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/86162e2766a8,41649,1733427915382 already deleted, retry=false 2024-12-05T19:45:20,333 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 86162e2766a8,41649,1733427915382 expired; onlineServers=0 2024-12-05T19:45:20,333 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '86162e2766a8,33193,1733427914584' ***** 2024-12-05T19:45:20,333 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T19:45:20,333 INFO [M:0;86162e2766a8:33193 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T19:45:20,333 INFO [M:0;86162e2766a8:33193 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T19:45:20,333 DEBUG [M:0;86162e2766a8:33193 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T19:45:20,333 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T19:45:20,334 DEBUG [M:0;86162e2766a8:33193 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T19:45:20,334 DEBUG [master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.large.0-1733427916553 {}] cleaner.HFileCleaner(306): Exit Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.large.0-1733427916553,5,FailOnTimeoutGroup] 2024-12-05T19:45:20,334 DEBUG [master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.small.0-1733427916554 {}] cleaner.HFileCleaner(306): Exit Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.small.0-1733427916554,5,FailOnTimeoutGroup] 2024-12-05T19:45:20,334 INFO [M:0;86162e2766a8:33193 {}] hbase.ChoreService(370): Chore service for: master/86162e2766a8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T19:45:20,334 INFO [M:0;86162e2766a8:33193 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T19:45:20,334 DEBUG [M:0;86162e2766a8:33193 {}] master.HMaster(1795): Stopping service threads 2024-12-05T19:45:20,334 INFO [M:0;86162e2766a8:33193 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T19:45:20,335 INFO [M:0;86162e2766a8:33193 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T19:45:20,335 INFO [M:0;86162e2766a8:33193 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T19:45:20,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T19:45:20,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,336 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T19:45:20,336 DEBUG [M:0;86162e2766a8:33193 {}] zookeeper.ZKUtil(347): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T19:45:20,336 WARN [M:0;86162e2766a8:33193 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T19:45:20,337 INFO [M:0;86162e2766a8:33193 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/.lastflushedseqids 2024-12-05T19:45:20,350 INFO [M:0;86162e2766a8:33193 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T19:45:20,350 INFO [M:0;86162e2766a8:33193 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T19:45:20,351 DEBUG [M:0;86162e2766a8:33193 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T19:45:20,351 INFO [M:0;86162e2766a8:33193 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:20,351 DEBUG [M:0;86162e2766a8:33193 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:20,351 DEBUG [M:0;86162e2766a8:33193 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T19:45:20,351 DEBUG [M:0;86162e2766a8:33193 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:20,351 INFO [M:0;86162e2766a8:33193 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=35.44 KB heapSize=43.91 KB 2024-12-05T19:45:20,362 DEBUG [M:0;86162e2766a8:33193 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ba8b3c47f57c444f971d341ea88f43b6 2024-12-05T19:45:20,376 DEBUG [M:0;86162e2766a8:33193 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba8b3c47f57c444f971d341ea88f43b6 is 82, key is hbase:meta,,1/info:regioninfo/1733427917391/Put/seqid=0 2024-12-05T19:45:20,377 INFO [M:0;86162e2766a8:33193 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=86 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba8b3c47f57c444f971d341ea88f43b6 2024-12-05T19:45:20,385 DEBUG [M:0;86162e2766a8:33193 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 537f28fa1bfc4b22b62bae1b9afbfff4 2024-12-05T19:45:20,400 DEBUG [M:0;86162e2766a8:33193 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/537f28fa1bfc4b22b62bae1b9afbfff4 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733427918129/Put/seqid=0 2024-12-05T19:45:20,401 INFO [M:0;86162e2766a8:33193 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.82 KB at sequenceid=86 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/537f28fa1bfc4b22b62bae1b9afbfff4 2024-12-05T19:45:20,409 DEBUG [M:0;86162e2766a8:33193 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 66f0dbdcb5514f0588d6ea3f31aafafc 2024-12-05T19:45:20,423 DEBUG [M:0;86162e2766a8:33193 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/66f0dbdcb5514f0588d6ea3f31aafafc is 69, key is 86162e2766a8,40199,1733427915541/rs:state/1733427916639/Put/seqid=0 2024-12-05T19:45:20,424 INFO [M:0;86162e2766a8:33193 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=86 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/66f0dbdcb5514f0588d6ea3f31aafafc 2024-12-05T19:45:20,426 DEBUG [M:0;86162e2766a8:33193 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba8b3c47f57c444f971d341ea88f43b6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ba8b3c47f57c444f971d341ea88f43b6 2024-12-05T19:45:20,428 INFO [M:0;86162e2766a8:33193 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ba8b3c47f57c444f971d341ea88f43b6, entries=8, sequenceid=86, filesize=5.5 K 2024-12-05T19:45:20,428 DEBUG [M:0;86162e2766a8:33193 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/537f28fa1bfc4b22b62bae1b9afbfff4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/537f28fa1bfc4b22b62bae1b9afbfff4 2024-12-05T19:45:20,430 INFO [M:0;86162e2766a8:33193 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/537f28fa1bfc4b22b62bae1b9afbfff4, entries=9, sequenceid=86, filesize=6.5 K 2024-12-05T19:45:20,430 DEBUG [M:0;86162e2766a8:33193 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/66f0dbdcb5514f0588d6ea3f31aafafc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/66f0dbdcb5514f0588d6ea3f31aafafc 2024-12-05T19:45:20,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:20,431 INFO [RS:0;86162e2766a8:41649 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T19:45:20,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41649-0x10063b69eeb0001, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:20,431 INFO [RS:0;86162e2766a8:41649 {}] regionserver.HRegionServer(1031): Exiting; stopping=86162e2766a8,41649,1733427915382; zookeeper connection closed. 2024-12-05T19:45:20,431 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c1942e6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c1942e6 2024-12-05T19:45:20,432 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-05T19:45:20,432 INFO [M:0;86162e2766a8:33193 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/66f0dbdcb5514f0588d6ea3f31aafafc, entries=2, sequenceid=86, filesize=5.1 K 2024-12-05T19:45:20,433 INFO [M:0;86162e2766a8:33193 {}] regionserver.HRegion(3140): Finished flush of dataSize ~35.44 KB/36287, heapSize ~43.62 KB/44664, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=86, compaction requested=false 2024-12-05T19:45:20,435 INFO [M:0;86162e2766a8:33193 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:20,435 DEBUG [M:0;86162e2766a8:33193 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733427920350Disabling compacts and flushes for region at 1733427920350Disabling writes for close at 1733427920351 (+1 ms)Obtaining lock to block concurrent updates at 1733427920351Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733427920351Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=36287, getHeapSize=44904, getOffHeapSize=0, getCellsCount=101 at 1733427920352 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733427920352Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733427920352Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733427920375 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733427920375Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733427920379 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733427920399 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733427920399Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733427920403 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733427920423 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733427920423Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5486f2da: reopening flushed file at 1733427920426 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bf865de: reopening flushed file at 1733427920428 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2912ac89: reopening flushed file at 1733427920430 (+2 ms)Finished flush of dataSize ~35.44 KB/36287, heapSize ~43.62 KB/44664, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=86, compaction requested=false at 1733427920433 (+3 ms)Writing region close event to WAL at 1733427920435 (+2 ms)Closed at 1733427920435 2024-12-05T19:45:20,437 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T19:45:20,437 INFO [M:0;86162e2766a8:33193 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T19:45:20,437 INFO [M:0;86162e2766a8:33193 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33193 2024-12-05T19:45:20,437 INFO [M:0;86162e2766a8:33193 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T19:45:20,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:20,540 INFO [M:0;86162e2766a8:33193 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T19:45:20,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33193-0x10063b69eeb0000, quorum=127.0.0.1:59462, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:20,545 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T19:45:20,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-05T19:45:20,566 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:20,575 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/86162e2766a8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733427920205 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/86162e2766a8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733427918507 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733427920208 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/86162e2766a8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733427920039 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/86162e2766a8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=681 (was 743), ProcessCount=11 (was 11), AvailableMemoryMB=9031 (was 9361) 2024-12-05T19:45:20,581 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=681, ProcessCount=11, AvailableMemoryMB=9031 2024-12-05T19:45:20,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/hadoop-log-dir so I do NOT create it in target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c 2024-12-05T19:45:20,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/hadoop-tmp-dir so I do NOT create it in target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c 2024-12-05T19:45:20,581 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:20,585 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/cluster_f9a2dc12-8408-0ff2-e692-99c0bc64d46b, deleteOnExit=true 2024-12-05T19:45:20,587 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/cluster_f9a2dc12-8408-0ff2-e692-99c0bc64d46b/zookeeper_0, clientPort=50580, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/cluster_f9a2dc12-8408-0ff2-e692-99c0bc64d46b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/cluster_f9a2dc12-8408-0ff2-e692-99c0bc64d46b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T19:45:20,588 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50580 2024-12-05T19:45:20,602 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 with version=8 2024-12-05T19:45:20,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/hbase-staging 2024-12-05T19:45:20,605 INFO [Time-limited test {}] client.ConnectionUtils(128): master/86162e2766a8:0 server-side Connection retries=45 2024-12-05T19:45:20,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:20,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:20,606 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T19:45:20,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:20,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T19:45:20,606 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T19:45:20,606 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T19:45:20,607 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36249 2024-12-05T19:45:20,608 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36249 connecting to ZooKeeper ensemble=127.0.0.1:50580 2024-12-05T19:45:20,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:362490x0, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T19:45:20,615 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36249-0x10063b6b7b10000 connected 2024-12-05T19:45:20,632 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:20,632 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:20,633 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:20,634 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7, hbase.cluster.distributed=false 2024-12-05T19:45:20,635 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T19:45:20,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36249 2024-12-05T19:45:20,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36249 2024-12-05T19:45:20,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36249 2024-12-05T19:45:20,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36249 2024-12-05T19:45:20,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36249 2024-12-05T19:45:20,654 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/86162e2766a8:0 server-side Connection retries=45 2024-12-05T19:45:20,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:20,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:20,654 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T19:45:20,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:20,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T19:45:20,654 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T19:45:20,654 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T19:45:20,655 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46391 2024-12-05T19:45:20,657 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46391 connecting to ZooKeeper ensemble=127.0.0.1:50580 2024-12-05T19:45:20,657 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:20,657 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:20,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463910x0, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T19:45:20,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46391-0x10063b6b7b10001 connected 2024-12-05T19:45:20,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:20,661 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T19:45:20,662 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.cache 2024-12-05T19:45:20,662 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T19:45:20,662 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T19:45:20,662 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T19:45:20,662 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T19:45:20,663 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T19:45:20,663 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T19:45:20,663 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T19:45:20,663 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.persistence is complete. 2024-12-05T19:45:20,663 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T19:45:20,664 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T19:45:20,665 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T19:45:20,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46391 2024-12-05T19:45:20,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46391 2024-12-05T19:45:20,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46391 2024-12-05T19:45:20,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46391 2024-12-05T19:45:20,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46391 2024-12-05T19:45:20,690 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/86162e2766a8:0 server-side Connection retries=45 2024-12-05T19:45:20,690 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:20,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:20,691 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T19:45:20,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:20,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T19:45:20,691 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T19:45:20,691 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T19:45:20,692 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38283 2024-12-05T19:45:20,694 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38283 connecting to ZooKeeper ensemble=127.0.0.1:50580 2024-12-05T19:45:20,694 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:20,695 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:20,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382830x0, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T19:45:20,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:382830x0, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:20,699 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38283-0x10063b6b7b10002 connected 2024-12-05T19:45:20,699 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T19:45:20,702 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.cache 2024-12-05T19:45:20,702 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T19:45:20,702 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T19:45:20,702 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T19:45:20,703 INFO [Thread-122 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T19:45:20,703 WARN [Thread-122 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T19:45:20,703 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T19:45:20,703 INFO [Thread-122 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T19:45:20,703 INFO [Thread-122 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.persistence is complete. 2024-12-05T19:45:20,704 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T19:45:20,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T19:45:20,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T19:45:20,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38283 2024-12-05T19:45:20,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38283 2024-12-05T19:45:20,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38283 2024-12-05T19:45:20,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38283 2024-12-05T19:45:20,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38283 2024-12-05T19:45:20,728 DEBUG [M:0;86162e2766a8:36249 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;86162e2766a8:36249 2024-12-05T19:45:20,728 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/86162e2766a8,36249,1733427920604 2024-12-05T19:45:20,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:20,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:20,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:20,731 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/86162e2766a8,36249,1733427920604 2024-12-05T19:45:20,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T19:45:20,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T19:45:20,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,734 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T19:45:20,735 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/86162e2766a8,36249,1733427920604 from backup master directory 2024-12-05T19:45:20,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/86162e2766a8,36249,1733427920604 2024-12-05T19:45:20,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:20,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:20,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T19:45:20,737 WARN [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T19:45:20,737 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=86162e2766a8,36249,1733427920604 2024-12-05T19:45:20,739 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/hbase.id] with ID: ebb45bdd-5479-45bb-a772-58c8bda1a76c 2024-12-05T19:45:20,739 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/.tmp/hbase.id 2024-12-05T19:45:20,743 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/hbase.id] 2024-12-05T19:45:20,782 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:20,782 INFO [master/86162e2766a8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T19:45:20,784 INFO [master/86162e2766a8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-05T19:45:20,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,792 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T19:45:20,793 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T19:45:20,793 INFO [master/86162e2766a8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T19:45:20,815 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store 2024-12-05T19:45:20,822 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:20,822 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T19:45:20,822 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:20,823 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:20,823 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T19:45:20,823 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:20,823 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:20,823 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733427920822Disabling compacts and flushes for region at 1733427920822Disabling writes for close at 1733427920823 (+1 ms)Writing region close event to WAL at 1733427920823Closed at 1733427920823 2024-12-05T19:45:20,823 WARN [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/.initializing 2024-12-05T19:45:20,823 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/WALs/86162e2766a8,36249,1733427920604 2024-12-05T19:45:20,829 INFO [master/86162e2766a8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T19:45:20,833 INFO [master/86162e2766a8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=86162e2766a8%2C36249%2C1733427920604, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/WALs/86162e2766a8,36249,1733427920604, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/oldWALs, maxLogs=10 2024-12-05T19:45:20,853 INFO [master/86162e2766a8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/WALs/86162e2766a8,36249,1733427920604/86162e2766a8%2C36249%2C1733427920604.1733427920834 2024-12-05T19:45:20,854 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T19:45:20,854 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T19:45:20,854 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:20,855 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,855 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T19:45:20,861 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:20,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:20,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T19:45:20,866 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:20,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:20,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T19:45:20,871 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:20,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:20,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T19:45:20,876 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:20,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:20,876 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,877 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,877 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,877 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,877 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,877 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T19:45:20,878 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T19:45:20,885 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T19:45:20,886 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58767200, jitterRate=-0.12430047988891602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T19:45:20,887 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733427920855Initializing all the Stores at 1733427920855Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427920855Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427920856 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427920856Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427920856Cleaning up temporary data from old regions at 1733427920877 (+21 ms)Region opened successfully at 1733427920887 (+10 ms) 2024-12-05T19:45:20,887 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T19:45:20,892 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1616c9fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=86162e2766a8/172.17.0.2:0 2024-12-05T19:45:20,893 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T19:45:20,893 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T19:45:20,893 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T19:45:20,894 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T19:45:20,894 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T19:45:20,894 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-05T19:45:20,894 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T19:45:20,896 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T19:45:20,898 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T19:45:20,899 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T19:45:20,899 INFO [master/86162e2766a8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T19:45:20,900 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T19:45:20,901 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T19:45:20,902 INFO [master/86162e2766a8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T19:45:20,903 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T19:45:20,904 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T19:45:20,906 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T19:45:20,907 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T19:45:20,910 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T19:45:20,911 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T19:45:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,915 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=86162e2766a8,36249,1733427920604, sessionid=0x10063b6b7b10000, setting cluster-up flag (Was=false) 2024-12-05T19:45:20,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,925 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T19:45:20,926 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=86162e2766a8,36249,1733427920604 2024-12-05T19:45:20,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:20,937 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T19:45:20,939 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=86162e2766a8,36249,1733427920604 2024-12-05T19:45:20,939 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T19:45:20,941 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T19:45:20,941 INFO [master/86162e2766a8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T19:45:20,942 INFO [master/86162e2766a8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T19:45:20,942 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 86162e2766a8,36249,1733427920604 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T19:45:20,944 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T19:45:20,944 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T19:45:20,944 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T19:45:20,944 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T19:45:20,944 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/86162e2766a8:0, corePoolSize=10, maxPoolSize=10 2024-12-05T19:45:20,944 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:20,944 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/86162e2766a8:0, corePoolSize=2, maxPoolSize=2 2024-12-05T19:45:20,944 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:20,946 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T19:45:20,946 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733427950946 2024-12-05T19:45:20,946 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T19:45:20,946 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T19:45:20,946 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T19:45:20,946 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T19:45:20,946 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:20,946 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T19:45:20,946 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T19:45:20,946 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T19:45:20,946 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T19:45:20,948 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:20,948 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T19:45:20,949 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T19:45:20,949 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T19:45:20,952 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T19:45:20,952 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T19:45:20,956 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.large.0-1733427920952,5,FailOnTimeoutGroup] 2024-12-05T19:45:20,956 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.small.0-1733427920956,5,FailOnTimeoutGroup] 2024-12-05T19:45:20,956 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:20,956 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T19:45:20,956 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:20,956 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:20,968 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T19:45:20,968 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 2024-12-05T19:45:20,976 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:20,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T19:45:20,986 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T19:45:20,986 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:20,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:20,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T19:45:20,991 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T19:45:20,991 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:20,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:20,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T19:45:20,995 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T19:45:20,996 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:20,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:20,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T19:45:21,000 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T19:45:21,000 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:21,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:21,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T19:45:21,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740 2024-12-05T19:45:21,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740 2024-12-05T19:45:21,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T19:45:21,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T19:45:21,002 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T19:45:21,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T19:45:21,011 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T19:45:21,011 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74140754, jitterRate=0.1047833263874054}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T19:45:21,012 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733427920977Initializing all the Stores at 1733427920977Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427920977Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427920980 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427920980Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427920980Cleaning up temporary data from old regions at 1733427921002 (+22 ms)Region opened successfully at 1733427921012 (+10 ms) 2024-12-05T19:45:21,012 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T19:45:21,012 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T19:45:21,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T19:45:21,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T19:45:21,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T19:45:21,013 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T19:45:21,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733427921012Disabling compacts and flushes for region at 1733427921012Disabling writes for close at 1733427921013 (+1 ms)Writing region close event to WAL at 1733427921013Closed at 1733427921013 2024-12-05T19:45:21,014 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T19:45:21,014 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T19:45:21,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T19:45:21,016 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T19:45:21,017 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T19:45:21,019 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(746): ClusterId : ebb45bdd-5479-45bb-a772-58c8bda1a76c 2024-12-05T19:45:21,019 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(746): ClusterId : ebb45bdd-5479-45bb-a772-58c8bda1a76c 2024-12-05T19:45:21,019 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T19:45:21,019 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T19:45:21,023 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T19:45:21,023 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T19:45:21,025 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T19:45:21,026 DEBUG [RS:0;86162e2766a8:46391 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112d72d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=86162e2766a8/172.17.0.2:0 2024-12-05T19:45:21,028 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T19:45:21,028 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T19:45:21,031 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T19:45:21,031 DEBUG [RS:1;86162e2766a8:38283 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c709a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=86162e2766a8/172.17.0.2:0 2024-12-05T19:45:21,040 DEBUG [RS:0;86162e2766a8:46391 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;86162e2766a8:46391 2024-12-05T19:45:21,040 INFO [RS:0;86162e2766a8:46391 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T19:45:21,040 INFO [RS:0;86162e2766a8:46391 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T19:45:21,040 DEBUG [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T19:45:21,041 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(2659): reportForDuty to master=86162e2766a8,36249,1733427920604 with port=46391, startcode=1733427920653 2024-12-05T19:45:21,042 DEBUG [RS:0;86162e2766a8:46391 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T19:45:21,045 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41699, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T19:45:21,045 DEBUG [RS:1;86162e2766a8:38283 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;86162e2766a8:38283 2024-12-05T19:45:21,046 INFO [RS:1;86162e2766a8:38283 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T19:45:21,046 INFO [RS:1;86162e2766a8:38283 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T19:45:21,046 DEBUG [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T19:45:21,046 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36249 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 86162e2766a8,46391,1733427920653 2024-12-05T19:45:21,046 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36249 {}] master.ServerManager(517): Registering regionserver=86162e2766a8,46391,1733427920653 2024-12-05T19:45:21,046 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(2659): reportForDuty to master=86162e2766a8,36249,1733427920604 with port=38283, startcode=1733427920690 2024-12-05T19:45:21,047 DEBUG [RS:1;86162e2766a8:38283 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T19:45:21,048 DEBUG [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 2024-12-05T19:45:21,048 DEBUG [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T19:45:21,048 DEBUG [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T19:45:21,049 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55811, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T19:45:21,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T19:45:21,050 DEBUG [RS:0;86162e2766a8:46391 {}] zookeeper.ZKUtil(111): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/86162e2766a8,46391,1733427920653 2024-12-05T19:45:21,050 WARN [RS:0;86162e2766a8:46391 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T19:45:21,051 INFO [RS:0;86162e2766a8:46391 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T19:45:21,051 DEBUG [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,46391,1733427920653 2024-12-05T19:45:21,051 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [86162e2766a8,46391,1733427920653] 2024-12-05T19:45:21,052 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36249 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 86162e2766a8,38283,1733427920690 2024-12-05T19:45:21,052 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36249 {}] master.ServerManager(517): Registering regionserver=86162e2766a8,38283,1733427920690 2024-12-05T19:45:21,054 DEBUG [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 2024-12-05T19:45:21,054 DEBUG [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T19:45:21,054 DEBUG [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T19:45:21,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T19:45:21,060 DEBUG [RS:1;86162e2766a8:38283 {}] zookeeper.ZKUtil(111): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/86162e2766a8,38283,1733427920690 2024-12-05T19:45:21,060 WARN [RS:1;86162e2766a8:38283 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T19:45:21,060 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [86162e2766a8,38283,1733427920690] 2024-12-05T19:45:21,060 INFO [RS:1;86162e2766a8:38283 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T19:45:21,061 DEBUG [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,38283,1733427920690 2024-12-05T19:45:21,061 INFO [RS:0;86162e2766a8:46391 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T19:45:21,065 INFO [RS:0;86162e2766a8:46391 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T19:45:21,065 INFO [RS:0;86162e2766a8:46391 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T19:45:21,065 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,066 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T19:45:21,066 INFO [RS:1;86162e2766a8:38283 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T19:45:21,066 INFO [RS:0;86162e2766a8:46391 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T19:45:21,067 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,067 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,067 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,067 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,067 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,067 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,067 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/86162e2766a8:0, corePoolSize=2, maxPoolSize=2 2024-12-05T19:45:21,067 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,068 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,068 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,068 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,068 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,068 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,068 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:21,068 DEBUG [RS:0;86162e2766a8:46391 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:21,069 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,069 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,069 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,069 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,069 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,069 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,46391,1733427920653-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T19:45:21,070 INFO [RS:1;86162e2766a8:38283 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T19:45:21,070 INFO [RS:1;86162e2766a8:38283 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T19:45:21,070 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,070 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T19:45:21,071 INFO [RS:1;86162e2766a8:38283 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T19:45:21,071 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,071 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,071 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,071 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,071 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,072 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,072 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/86162e2766a8:0, corePoolSize=2, maxPoolSize=2 2024-12-05T19:45:21,072 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,072 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,072 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,072 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,072 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,072 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:21,072 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:21,073 DEBUG [RS:1;86162e2766a8:38283 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:21,073 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,073 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,074 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,074 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,074 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,074 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,38283,1733427920690-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T19:45:21,089 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T19:45:21,089 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,38283,1733427920690-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,089 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T19:45:21,090 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,090 INFO [RS:1;86162e2766a8:38283 {}] regionserver.Replication(171): 86162e2766a8,38283,1733427920690 started 2024-12-05T19:45:21,090 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,46391,1733427920653-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,090 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,090 INFO [RS:0;86162e2766a8:46391 {}] regionserver.Replication(171): 86162e2766a8,46391,1733427920653 started 2024-12-05T19:45:21,105 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,105 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,105 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(1482): Serving as 86162e2766a8,38283,1733427920690, RpcServer on 86162e2766a8/172.17.0.2:38283, sessionid=0x10063b6b7b10002 2024-12-05T19:45:21,105 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(1482): Serving as 86162e2766a8,46391,1733427920653, RpcServer on 86162e2766a8/172.17.0.2:46391, sessionid=0x10063b6b7b10001 2024-12-05T19:45:21,105 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T19:45:21,105 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T19:45:21,105 DEBUG [RS:0;86162e2766a8:46391 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 86162e2766a8,46391,1733427920653 2024-12-05T19:45:21,105 DEBUG [RS:1;86162e2766a8:38283 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 86162e2766a8,38283,1733427920690 2024-12-05T19:45:21,105 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,46391,1733427920653' 2024-12-05T19:45:21,105 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,38283,1733427920690' 2024-12-05T19:45:21,105 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T19:45:21,105 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T19:45:21,106 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T19:45:21,106 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T19:45:21,107 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T19:45:21,107 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T19:45:21,107 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T19:45:21,107 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T19:45:21,107 DEBUG [RS:1;86162e2766a8:38283 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 86162e2766a8,38283,1733427920690 2024-12-05T19:45:21,107 DEBUG [RS:0;86162e2766a8:46391 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 86162e2766a8,46391,1733427920653 2024-12-05T19:45:21,107 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,38283,1733427920690' 2024-12-05T19:45:21,107 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,46391,1733427920653' 2024-12-05T19:45:21,107 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T19:45:21,107 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T19:45:21,108 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T19:45:21,108 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T19:45:21,108 DEBUG [RS:0;86162e2766a8:46391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T19:45:21,108 INFO [RS:0;86162e2766a8:46391 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T19:45:21,108 INFO [RS:0;86162e2766a8:46391 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T19:45:21,108 DEBUG [RS:1;86162e2766a8:38283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T19:45:21,109 INFO [RS:1;86162e2766a8:38283 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T19:45:21,109 INFO [RS:1;86162e2766a8:38283 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T19:45:21,167 WARN [86162e2766a8:36249 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T19:45:21,209 INFO [RS:1;86162e2766a8:38283 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T19:45:21,209 INFO [RS:0;86162e2766a8:46391 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T19:45:21,210 INFO [RS:1;86162e2766a8:38283 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=86162e2766a8%2C38283%2C1733427920690, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,38283,1733427920690, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs, maxLogs=55 2024-12-05T19:45:21,210 INFO [RS:0;86162e2766a8:46391 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=86162e2766a8%2C46391%2C1733427920653, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,46391,1733427920653, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs, maxLogs=55 2024-12-05T19:45:21,230 INFO [RS:1;86162e2766a8:38283 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,38283,1733427920690/86162e2766a8%2C38283%2C1733427920690.1733427921212 2024-12-05T19:45:21,231 DEBUG [RS:1;86162e2766a8:38283 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T19:45:21,237 INFO [RS:0;86162e2766a8:46391 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,46391,1733427920653/86162e2766a8%2C46391%2C1733427920653.1733427921213 2024-12-05T19:45:21,238 DEBUG [RS:0;86162e2766a8:46391 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T19:45:21,418 DEBUG [86162e2766a8:36249 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-05T19:45:21,418 DEBUG [86162e2766a8:36249 {}] balancer.BalancerClusterState(204): Hosts are {86162e2766a8=0} racks are {/default-rack=0} 2024-12-05T19:45:21,419 DEBUG [86162e2766a8:36249 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T19:45:21,419 DEBUG [86162e2766a8:36249 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T19:45:21,419 DEBUG [86162e2766a8:36249 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T19:45:21,419 DEBUG [86162e2766a8:36249 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T19:45:21,419 INFO [86162e2766a8:36249 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T19:45:21,419 INFO [86162e2766a8:36249 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T19:45:21,419 DEBUG [86162e2766a8:36249 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T19:45:21,420 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=86162e2766a8,38283,1733427920690 2024-12-05T19:45:21,421 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 86162e2766a8,38283,1733427920690, state=OPENING 2024-12-05T19:45:21,423 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T19:45:21,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:21,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:21,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:21,425 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T19:45:21,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:21,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:21,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=86162e2766a8,38283,1733427920690}] 2024-12-05T19:45:21,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:21,579 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T19:45:21,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56905, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T19:45:21,584 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T19:45:21,584 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T19:45:21,585 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T19:45:21,586 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=86162e2766a8%2C38283%2C1733427920690.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,38283,1733427920690, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs, maxLogs=55 2024-12-05T19:45:21,606 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,38283,1733427920690/86162e2766a8%2C38283%2C1733427920690.meta.1733427921587.meta 2024-12-05T19:45:21,607 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T19:45:21,607 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T19:45:21,607 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T19:45:21,608 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T19:45:21,608 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T19:45:21,608 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T19:45:21,608 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:21,608 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T19:45:21,608 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T19:45:21,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T19:45:21,610 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T19:45:21,610 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:21,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:21,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T19:45:21,611 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T19:45:21,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:21,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:21,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T19:45:21,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T19:45:21,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:21,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:21,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T19:45:21,613 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T19:45:21,613 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:21,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T19:45:21,613 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T19:45:21,614 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740 2024-12-05T19:45:21,614 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740 2024-12-05T19:45:21,615 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T19:45:21,615 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T19:45:21,615 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T19:45:21,616 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T19:45:21,616 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59651191, jitterRate=-0.11112798750400543}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T19:45:21,616 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T19:45:21,617 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733427921609Writing region info on filesystem at 1733427921609Initializing all the Stores at 1733427921609Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427921609Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427921609Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427921609Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733427921609Cleaning up temporary data from old regions at 1733427921615 (+6 ms)Running coprocessor post-open hooks at 1733427921616 (+1 ms)Region opened successfully at 1733427921617 (+1 ms) 2024-12-05T19:45:21,618 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733427921579 2024-12-05T19:45:21,621 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T19:45:21,621 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T19:45:21,622 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,38283,1733427920690 2024-12-05T19:45:21,623 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 86162e2766a8,38283,1733427920690, state=OPEN 2024-12-05T19:45:21,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T19:45:21,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T19:45:21,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T19:45:21,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:21,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:21,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T19:45:21,625 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=86162e2766a8,38283,1733427920690 2024-12-05T19:45:21,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T19:45:21,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=86162e2766a8,38283,1733427920690 in 200 msec 2024-12-05T19:45:21,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T19:45:21,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 613 msec 2024-12-05T19:45:21,631 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T19:45:21,631 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T19:45:21,635 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T19:45:21,635 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=86162e2766a8,38283,1733427920690, seqNum=-1] 2024-12-05T19:45:21,636 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T19:45:21,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34217, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T19:45:21,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 706 msec 2024-12-05T19:45:21,648 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733427921648, completionTime=-1 2024-12-05T19:45:21,648 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-05T19:45:21,649 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T19:45:21,651 INFO [master/86162e2766a8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T19:45:21,651 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733427981651 2024-12-05T19:45:21,651 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733428041651 2024-12-05T19:45:21,651 INFO [master/86162e2766a8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-05T19:45:21,652 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T19:45:21,653 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,36249,1733427920604-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,653 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,36249,1733427920604-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,653 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,36249,1733427920604-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,653 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-86162e2766a8:36249, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,653 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,654 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,656 DEBUG [master/86162e2766a8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T19:45:21,660 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.923sec 2024-12-05T19:45:21,660 INFO [master/86162e2766a8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T19:45:21,660 INFO [master/86162e2766a8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T19:45:21,660 INFO [master/86162e2766a8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T19:45:21,660 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T19:45:21,661 INFO [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T19:45:21,661 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,36249,1733427920604-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T19:45:21,661 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,36249,1733427920604-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T19:45:21,662 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T19:45:21,662 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T19:45:21,662 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,36249,1733427920604-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:21,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T19:45:21,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 86162e2766a8,36249,-1 for getting cluster id 2024-12-05T19:45:21,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T19:45:21,721 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ebb45bdd-5479-45bb-a772-58c8bda1a76c' 2024-12-05T19:45:21,721 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T19:45:21,721 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ebb45bdd-5479-45bb-a772-58c8bda1a76c" 2024-12-05T19:45:21,722 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49180a5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T19:45:21,722 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [86162e2766a8,36249,-1] 2024-12-05T19:45:21,722 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T19:45:21,722 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:21,724 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T19:45:21,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T19:45:21,726 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T19:45:21,728 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=86162e2766a8,38283,1733427920690, seqNum=-1] 2024-12-05T19:45:21,728 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T19:45:21,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T19:45:21,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=86162e2766a8,36249,1733427920604 2024-12-05T19:45:21,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T19:45:21,734 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 86162e2766a8,36249,1733427920604 2024-12-05T19:45:21,734 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ee16b4 2024-12-05T19:45:21,734 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T19:45:21,736 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T19:45:21,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36249 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733427921732', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T19:45:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36249 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733427921732 2024-12-05T19:45:21,740 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733427921732 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T19:45:21,740 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:21,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36249 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733427921732" procId is: 4 2024-12-05T19:45:21,741 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733427921732 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T19:45:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T19:45:21,754 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733427921732-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3ef2e4eb9a86b2ef6359f7a33978fa77, NAME => 'testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733427921732', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 2024-12-05T19:45:21,762 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733427921732-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:21,762 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733427921732-pool-0 {}] regionserver.HRegion(1722): Closing 3ef2e4eb9a86b2ef6359f7a33978fa77, disabling compactions & flushes 2024-12-05T19:45:21,762 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733427921732-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:21,762 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733427921732-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:21,762 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733427921732-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. after waiting 0 ms 2024-12-05T19:45:21,762 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733427921732-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:21,762 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733427921732-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:21,762 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733427921732-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3ef2e4eb9a86b2ef6359f7a33978fa77: Waiting for close lock at 1733427921762Disabling compacts and flushes for region at 1733427921762Disabling writes for close at 1733427921762Writing region close event to WAL at 1733427921762Closed at 1733427921762 2024-12-05T19:45:21,763 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733427921732 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T19:45:21,764 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733427921763"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733427921763"}]},"ts":"1733427921763"} 2024-12-05T19:45:21,766 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T19:45:21,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733427921732 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T19:45:21,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733427921732","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733427921767"}]},"ts":"1733427921767"} 2024-12-05T19:45:21,770 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733427921732, state=ENABLING in hbase:meta 2024-12-05T19:45:21,770 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {86162e2766a8=0} racks are {/default-rack=0} 2024-12-05T19:45:21,771 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T19:45:21,771 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T19:45:21,771 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T19:45:21,771 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T19:45:21,771 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T19:45:21,771 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T19:45:21,771 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T19:45:21,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN}] 2024-12-05T19:45:21,772 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN 2024-12-05T19:45:21,773 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN; state=OFFLINE, location=86162e2766a8,46391,1733427920653; forceNewPlan=false, retain=false 2024-12-05T19:45:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T19:45:21,923 INFO [86162e2766a8:36249 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T19:45:21,924 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3ef2e4eb9a86b2ef6359f7a33978fa77, regionState=OPENING, regionLocation=86162e2766a8,46391,1733427920653 2024-12-05T19:45:21,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN because future has completed 2024-12-05T19:45:21,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ef2e4eb9a86b2ef6359f7a33978fa77, server=86162e2766a8,46391,1733427920653}] 2024-12-05T19:45:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T19:45:22,081 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T19:45:22,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T19:45:22,085 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:22,086 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ef2e4eb9a86b2ef6359f7a33978fa77, NAME => 'testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77.', STARTKEY => '', ENDKEY => ''} 2024-12-05T19:45:22,086 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733427921732 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,086 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:22,086 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,087 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,088 INFO [StoreOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,092 INFO [StoreOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ef2e4eb9a86b2ef6359f7a33978fa77 columnFamilyName family 2024-12-05T19:45:22,092 DEBUG [StoreOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:22,093 INFO [StoreOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-1 {}] regionserver.HStore(327): Store=3ef2e4eb9a86b2ef6359f7a33978fa77/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:22,093 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,094 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,094 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,094 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,095 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,096 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,102 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T19:45:22,103 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3ef2e4eb9a86b2ef6359f7a33978fa77; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59215492, jitterRate=-0.11762040853500366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T19:45:22,103 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:22,104 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3ef2e4eb9a86b2ef6359f7a33978fa77: Running coprocessor pre-open hook at 1733427922087Writing region info on filesystem at 1733427922087Initializing all the Stores at 1733427922087Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427922087Cleaning up temporary data from old regions at 1733427922095 (+8 ms)Running coprocessor post-open hooks at 1733427922103 (+8 ms)Region opened successfully at 1733427922104 (+1 ms) 2024-12-05T19:45:22,105 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77., pid=6, masterSystemTime=1733427922080 2024-12-05T19:45:22,108 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:22,108 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:22,108 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3ef2e4eb9a86b2ef6359f7a33978fa77, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,46391,1733427920653 2024-12-05T19:45:22,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ef2e4eb9a86b2ef6359f7a33978fa77, server=86162e2766a8,46391,1733427920653 because future has completed 2024-12-05T19:45:22,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T19:45:22,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ef2e4eb9a86b2ef6359f7a33978fa77, server=86162e2766a8,46391,1733427920653 in 184 msec 2024-12-05T19:45:22,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T19:45:22,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN in 343 msec 2024-12-05T19:45:22,117 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733427921732 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T19:45:22,117 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733427921732","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733427922117"}]},"ts":"1733427922117"} 2024-12-05T19:45:22,120 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733427921732, state=ENABLED in hbase:meta 2024-12-05T19:45:22,120 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733427921732 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T19:45:22,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733427921732 in 383 msec 2024-12-05T19:45:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T19:45:22,367 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733427921732 completed 2024-12-05T19:45:22,367 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733427921732 get assigned. Timeout = 60000ms 2024-12-05T19:45:22,367 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T19:45:22,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733427921732 assigned to meta. Checking AM states. 2024-12-05T19:45:22,372 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T19:45:22,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733427921732 assigned. 2024-12-05T19:45:22,375 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733427921732', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77., hostname=86162e2766a8,46391,1733427920653, seqNum=2] 2024-12-05T19:45:22,375 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T19:45:22,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T19:45:22,382 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3ef2e4eb9a86b2ef6359f7a33978fa77 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-05T19:45:22,393 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 09d3acbbd7c0483c842346f25dc80c43 2024-12-05T19:45:22,413 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/.tmp/family/09d3acbbd7c0483c842346f25dc80c43 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-05T19:45:22,414 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/.tmp/family/09d3acbbd7c0483c842346f25dc80c43 2024-12-05T19:45:22,417 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/.tmp/family/09d3acbbd7c0483c842346f25dc80c43 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43 2024-12-05T19:45:22,419 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43, delay=996 ms 2024-12-05T19:45:22,421 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T19:45:22,421 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 3ef2e4eb9a86b2ef6359f7a33978fa77 in 39ms, sequenceid=6, compaction requested=false 2024-12-05T19:45:22,422 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733427921732' 2024-12-05T19:45:22,422 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3ef2e4eb9a86b2ef6359f7a33978fa77: 2024-12-05T19:45:22,859 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T19:45:23,416 DEBUG [hfile-prefetch-1733427920039 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 09d3acbbd7c0483c842346f25dc80c43_0. 2024-12-05T19:45:23,417 DEBUG [hfile-prefetch-1733427920039 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 09d3acbbd7c0483c842346f25dc80c43_117. 2024-12-05T19:45:23,418 DEBUG [hfile-prefetch-1733427920039 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43, with total blocks 2, and data blocks 1 2024-12-05T19:45:23,418 DEBUG [hfile-prefetch-1733427920039 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-05T19:45:23,418 DEBUG [hfile-prefetch-1733427920039 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-05T19:45:23,418 DEBUG [hfile-prefetch-1733427920039 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43. Will sleep 100ms and try the verification again. 2024-12-05T19:45:23,423 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;86162e2766a8:46391,5,FailOnTimeoutGroup] 2024-12-05T19:45:23,423 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '86162e2766a8,46391,1733427920653' ***** 2024-12-05T19:45:23,423 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-05T19:45:23,423 INFO [RS:0;86162e2766a8:46391 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T19:45:23,424 INFO [RS:0;86162e2766a8:46391 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.persistence 2024-12-05T19:45:23,424 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T19:45:23,424 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T19:45:23,424 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T19:45:23,424 INFO [RS:0;86162e2766a8:46391 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T19:45:23,424 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T19:45:23,434 DEBUG [RS:0;86162e2766a8:46391 {}] bucket.BucketCache(1487): Thread RS:0;86162e2766a8:46391 started persisting bucket cache to file 2024-12-05T19:45:23,434 DEBUG [RS:0;86162e2766a8:46391 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:23,434 DEBUG [RS:0;86162e2766a8:46391 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:23,439 DEBUG [RS:0;86162e2766a8:46391 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-05T19:45:23,439 DEBUG [RS:0;86162e2766a8:46391 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733427923434 2024-12-05T19:45:23,439 DEBUG [RS:0;86162e2766a8:46391 {}] bucket.BucketCache(1509): Thread RS:0;86162e2766a8:46391 finished persisting bucket cache to file, renaming 2024-12-05T19:45:23,439 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T19:45:23,439 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T19:45:23,439 INFO [RS:0;86162e2766a8:46391 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T19:45:23,439 INFO [RS:0;86162e2766a8:46391 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T19:45:23,439 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(3091): Received CLOSE for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,440 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(959): stopping server 86162e2766a8,46391,1733427920653 2024-12-05T19:45:23,440 INFO [RS:0;86162e2766a8:46391 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T19:45:23,440 INFO [RS:0;86162e2766a8:46391 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;86162e2766a8:46391. 2024-12-05T19:45:23,440 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T19:45:23,440 DEBUG [RS:0;86162e2766a8:46391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T19:45:23,440 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3ef2e4eb9a86b2ef6359f7a33978fa77, disabling compactions & flushes 2024-12-05T19:45:23,440 DEBUG [RS:0;86162e2766a8:46391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:23,440 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:23,440 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:23,440 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. after waiting 0 ms 2024-12-05T19:45:23,440 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:23,441 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T19:45:23,441 DEBUG [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(1325): Online Regions={3ef2e4eb9a86b2ef6359f7a33978fa77=testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77.} 2024-12-05T19:45:23,441 DEBUG [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(1351): Waiting on 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,441 DEBUG [StoreFileCloser-3ef2e4eb9a86b2ef6359f7a33978fa77-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43 2024-12-05T19:45:23,445 WARN [hfile-prefetch-1733427920039 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-05T19:45:23,446 DEBUG [hfile-prefetch-1733427920039 {}] hfile.PrefetchExecutor(121): Prefetch completed for 09d3acbbd7c0483c842346f25dc80c43 2024-12-05T19:45:23,448 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T19:45:23,448 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:23,448 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3ef2e4eb9a86b2ef6359f7a33978fa77: Waiting for close lock at 1733427923440Running coprocessor pre-close hooks at 1733427923440Disabling compacts and flushes for region at 1733427923440Disabling writes for close at 1733427923440Writing region close event to WAL at 1733427923442 (+2 ms)Running coprocessor post-close hooks at 1733427923448 (+6 ms)Closed at 1733427923448 2024-12-05T19:45:23,448 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:23,471 INFO [regionserver/86162e2766a8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:23,505 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T19:45:23,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T19:45:23,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:23,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:23,508 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T19:45:23,508 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733427923505 2024-12-05T19:45:23,508 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T19:45:23,641 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(976): stopping server 86162e2766a8,46391,1733427920653; all regions closed. 2024-12-05T19:45:23,643 DEBUG [RS:0;86162e2766a8:46391 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs 2024-12-05T19:45:23,643 INFO [RS:0;86162e2766a8:46391 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 86162e2766a8%2C46391%2C1733427920653:(num 1733427921213) 2024-12-05T19:45:23,644 DEBUG [RS:0;86162e2766a8:46391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:23,644 INFO [RS:0;86162e2766a8:46391 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:23,644 INFO [RS:0;86162e2766a8:46391 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T19:45:23,644 INFO [RS:0;86162e2766a8:46391 {}] hbase.ChoreService(370): Chore service for: regionserver/86162e2766a8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T19:45:23,644 INFO [RS:0;86162e2766a8:46391 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T19:45:23,644 INFO [RS:0;86162e2766a8:46391 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T19:45:23,644 INFO [regionserver/86162e2766a8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T19:45:23,644 INFO [RS:0;86162e2766a8:46391 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T19:45:23,644 INFO [RS:0;86162e2766a8:46391 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T19:45:23,644 INFO [RS:0;86162e2766a8:46391 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46391 2024-12-05T19:45:23,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/86162e2766a8,46391,1733427920653 2024-12-05T19:45:23,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T19:45:23,649 INFO [RS:0;86162e2766a8:46391 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T19:45:23,651 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [86162e2766a8,46391,1733427920653] 2024-12-05T19:45:23,653 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/86162e2766a8,46391,1733427920653 already deleted, retry=false 2024-12-05T19:45:23,653 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 86162e2766a8,46391,1733427920653 on 86162e2766a8,36249,1733427920604 2024-12-05T19:45:23,658 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 86162e2766a8,46391,1733427920653, splitWal=true, meta=false 2024-12-05T19:45:23,661 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 86162e2766a8,46391,1733427920653, splitWal=true, meta=false 2024-12-05T19:45:23,661 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 86162e2766a8,46391,1733427920653 (carryingMeta=false) 86162e2766a8,46391,1733427920653/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2030cdb8[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-05T19:45:23,662 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 86162e2766a8,46391,1733427920653 had 1 regions 2024-12-05T19:45:23,663 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 86162e2766a8,46391,1733427920653, splitWal=true, meta=false, isMeta: false 2024-12-05T19:45:23,664 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,46391,1733427920653-splitting 2024-12-05T19:45:23,664 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,46391,1733427920653-splitting dir is empty, no logs to split. 2024-12-05T19:45:23,664 INFO [PEWorker-2 {}] master.SplitWALManager(105): 86162e2766a8,46391,1733427920653 WAL count=0, meta=false 2024-12-05T19:45:23,666 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,46391,1733427920653-splitting dir is empty, no logs to split. 2024-12-05T19:45:23,666 INFO [PEWorker-2 {}] master.SplitWALManager(105): 86162e2766a8,46391,1733427920653 WAL count=0, meta=false 2024-12-05T19:45:23,666 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 86162e2766a8,46391,1733427920653 WAL splitting is done? wals=0, meta=false 2024-12-05T19:45:23,667 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 86162e2766a8,46391,1733427920653 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,46391,1733427920653-splitting does not exist 2024-12-05T19:45:23,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN}] 2024-12-05T19:45:23,670 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN 2024-12-05T19:45:23,670 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T19:45:23,751 INFO [RS:0;86162e2766a8:46391 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T19:45:23,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:23,751 INFO [RS:0;86162e2766a8:46391 {}] regionserver.HRegionServer(1031): Exiting; stopping=86162e2766a8,46391,1733427920653; zookeeper connection closed. 2024-12-05T19:45:23,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46391-0x10063b6b7b10001, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:23,752 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@213385ab {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@213385ab 2024-12-05T19:45:23,821 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=3ef2e4eb9a86b2ef6359f7a33978fa77, regionState=OPENING, regionLocation=86162e2766a8,38283,1733427920690 2024-12-05T19:45:23,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN because future has completed 2024-12-05T19:45:23,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ef2e4eb9a86b2ef6359f7a33978fa77, server=86162e2766a8,38283,1733427920690}] 2024-12-05T19:45:23,945 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/86162e2766a8:0 server-side Connection retries=45 2024-12-05T19:45:23,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:23,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:23,946 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T19:45:23,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T19:45:23,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T19:45:23,946 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T19:45:23,946 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T19:45:23,947 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39853 2024-12-05T19:45:23,949 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39853 connecting to ZooKeeper ensemble=127.0.0.1:50580 2024-12-05T19:45:23,949 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:23,949 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:23,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:398530x0, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T19:45:23,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39853-0x10063b6b7b10003 connected 2024-12-05T19:45:23,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-05T19:45:23,954 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-05T19:45:23,955 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T19:45:23,955 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.cache 2024-12-05T19:45:23,955 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T19:45:23,955 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T19:45:23,955 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T19:45:23,956 INFO [Thread-202 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T19:45:23,956 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T19:45:23,957 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T19:45:23,957 INFO [Thread-202 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-05T19:45:23,957 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T19:45:23,958 INFO [Thread-202 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-05T19:45:23,959 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T19:45:23,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39853 2024-12-05T19:45:23,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39853 2024-12-05T19:45:23,961 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39853 2024-12-05T19:45:23,961 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39853 2024-12-05T19:45:23,961 DEBUG [Thread-202 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-05T19:45:23,961 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39853 2024-12-05T19:45:23,962 INFO [Thread-202 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T19:45:23,962 INFO [Thread-202 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-05T19:45:23,962 INFO [Thread-202 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.persistence is complete. 2024-12-05T19:45:23,963 INFO [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(746): ClusterId : ebb45bdd-5479-45bb-a772-58c8bda1a76c 2024-12-05T19:45:23,963 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T19:45:23,965 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T19:45:23,965 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T19:45:23,967 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T19:45:23,968 DEBUG [RS:2;86162e2766a8:39853 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c17ad2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=86162e2766a8/172.17.0.2:0 2024-12-05T19:45:23,979 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:23,980 DEBUG [RS:2;86162e2766a8:39853 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;86162e2766a8:39853 2024-12-05T19:45:23,980 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ef2e4eb9a86b2ef6359f7a33978fa77, NAME => 'testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77.', STARTKEY => '', ENDKEY => ''} 2024-12-05T19:45:23,980 INFO [RS:2;86162e2766a8:39853 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T19:45:23,980 INFO [RS:2;86162e2766a8:39853 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T19:45:23,980 DEBUG [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T19:45:23,980 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733427921732 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,980 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T19:45:23,981 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,981 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,981 INFO [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(2659): reportForDuty to master=86162e2766a8,36249,1733427920604 with port=39853, startcode=1733427923945 2024-12-05T19:45:23,981 DEBUG [RS:2;86162e2766a8:39853 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T19:45:23,981 INFO [StoreOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,982 INFO [StoreOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ef2e4eb9a86b2ef6359f7a33978fa77 columnFamilyName family 2024-12-05T19:45:23,982 DEBUG [StoreOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T19:45:23,983 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54059, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T19:45:23,983 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36249 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 86162e2766a8,39853,1733427923945 2024-12-05T19:45:23,984 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36249 {}] master.ServerManager(517): Registering regionserver=86162e2766a8,39853,1733427923945 2024-12-05T19:45:23,984 DEBUG [StoreFileOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43, delay=971 ms 2024-12-05T19:45:23,985 DEBUG [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7 2024-12-05T19:45:23,985 DEBUG [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T19:45:23,985 DEBUG [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T19:45:23,986 DEBUG [StoreOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43 2024-12-05T19:45:23,986 INFO [StoreOpener-3ef2e4eb9a86b2ef6359f7a33978fa77-1 {}] regionserver.HStore(327): Store=3ef2e4eb9a86b2ef6359f7a33978fa77/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T19:45:23,986 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,987 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T19:45:23,987 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,987 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,988 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,988 DEBUG [RS:2;86162e2766a8:39853 {}] zookeeper.ZKUtil(111): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/86162e2766a8,39853,1733427923945 2024-12-05T19:45:23,988 WARN [RS:2;86162e2766a8:39853 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T19:45:23,988 INFO [RS:2;86162e2766a8:39853 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T19:45:23,988 DEBUG [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,39853,1733427923945 2024-12-05T19:45:23,988 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [86162e2766a8,39853,1733427923945] 2024-12-05T19:45:23,989 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,990 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3ef2e4eb9a86b2ef6359f7a33978fa77; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68240256, jitterRate=0.016859054565429688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T19:45:23,990 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:23,991 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3ef2e4eb9a86b2ef6359f7a33978fa77: Running coprocessor pre-open hook at 1733427923981Writing region info on filesystem at 1733427923981Initializing all the Stores at 1733427923981Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733427923981Cleaning up temporary data from old regions at 1733427923988 (+7 ms)Running coprocessor post-open hooks at 1733427923990 (+2 ms)Region opened successfully at 1733427923991 (+1 ms) 2024-12-05T19:45:23,991 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77., pid=9, masterSystemTime=1733427923977 2024-12-05T19:45:23,994 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:23,994 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:23,995 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=3ef2e4eb9a86b2ef6359f7a33978fa77, regionState=OPEN, openSeqNum=10, regionLocation=86162e2766a8,38283,1733427920690 2024-12-05T19:45:23,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ef2e4eb9a86b2ef6359f7a33978fa77, server=86162e2766a8,38283,1733427920690 because future has completed 2024-12-05T19:45:23,998 INFO [RS:2;86162e2766a8:39853 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T19:45:24,000 INFO [RS:2;86162e2766a8:39853 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T19:45:24,001 INFO [RS:2;86162e2766a8:39853 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T19:45:24,001 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-05T19:45:24,001 INFO [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T19:45:24,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ef2e4eb9a86b2ef6359f7a33978fa77, server=86162e2766a8,38283,1733427920690 in 174 msec 2024-12-05T19:45:24,002 INFO [RS:2;86162e2766a8:39853 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T19:45:24,002 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,002 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,002 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,002 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,002 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,002 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,002 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/86162e2766a8:0, corePoolSize=2, maxPoolSize=2 2024-12-05T19:45:24,002 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,002 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,002 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,003 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,003 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,003 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T19:45:24,003 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:24,003 DEBUG [RS:2;86162e2766a8:39853 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T19:45:24,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T19:45:24,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733427921732, region=3ef2e4eb9a86b2ef6359f7a33978fa77, ASSIGN in 333 msec 2024-12-05T19:45:24,003 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 86162e2766a8,46391,1733427920653 after splitting done 2024-12-05T19:45:24,005 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,005 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,005 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,005 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,005 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,005 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,39853,1733427923945-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T19:45:24,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 86162e2766a8,46391,1733427920653, splitWal=true, meta=false in 349 msec 2024-12-05T19:45:24,024 INFO [RS:2;86162e2766a8:39853 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T19:45:24,025 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,39853,1733427923945-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,025 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,025 INFO [RS:2;86162e2766a8:39853 {}] regionserver.Replication(171): 86162e2766a8,39853,1733427923945 started 2024-12-05T19:45:24,040 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T19:45:24,040 INFO [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(1482): Serving as 86162e2766a8,39853,1733427923945, RpcServer on 86162e2766a8/172.17.0.2:39853, sessionid=0x10063b6b7b10003 2024-12-05T19:45:24,040 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T19:45:24,040 DEBUG [RS:2;86162e2766a8:39853 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 86162e2766a8,39853,1733427923945 2024-12-05T19:45:24,040 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,39853,1733427923945' 2024-12-05T19:45:24,040 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T19:45:24,041 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T19:45:24,042 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T19:45:24,042 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T19:45:24,042 DEBUG [RS:2;86162e2766a8:39853 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 86162e2766a8,39853,1733427923945 2024-12-05T19:45:24,042 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,39853,1733427923945' 2024-12-05T19:45:24,042 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T19:45:24,042 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T19:45:24,043 DEBUG [RS:2;86162e2766a8:39853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T19:45:24,043 INFO [RS:2;86162e2766a8:39853 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T19:45:24,043 INFO [RS:2;86162e2766a8:39853 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T19:45:24,143 INFO [RS:2;86162e2766a8:39853 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T19:45:24,144 INFO [RS:2;86162e2766a8:39853 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=86162e2766a8%2C39853%2C1733427923945, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,39853,1733427923945, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs, maxLogs=55 2024-12-05T19:45:24,164 INFO [RS:2;86162e2766a8:39853 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/WALs/86162e2766a8,39853,1733427923945/86162e2766a8%2C39853%2C1733427923945.1733427924145 2024-12-05T19:45:24,164 DEBUG [RS:2;86162e2766a8:39853 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T19:45:24,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T19:45:24,541 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T19:45:24,541 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T19:45:24,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:24,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:24,541 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T19:45:24,541 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T19:45:24,541 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1701651951, stopped=false 2024-12-05T19:45:24,541 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=86162e2766a8,36249,1733427920604 2024-12-05T19:45:24,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:24,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:24,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T19:45:24,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:24,543 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T19:45:24,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:24,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:24,544 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T19:45:24,544 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T19:45:24,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:24,544 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:24,544 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:24,544 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '86162e2766a8,38283,1733427920690' ***** 2024-12-05T19:45:24,544 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T19:45:24,544 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T19:45:24,545 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '86162e2766a8,39853,1733427923945' ***** 2024-12-05T19:45:24,545 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T19:45:24,545 INFO [RS:1;86162e2766a8:38283 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T19:45:24,545 INFO [RS:1;86162e2766a8:38283 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.persistence 2024-12-05T19:45:24,545 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T19:45:24,545 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T19:45:24,545 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T19:45:24,545 INFO [RS:1;86162e2766a8:38283 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T19:45:24,545 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T19:45:24,546 DEBUG [RS:1;86162e2766a8:38283 {}] bucket.BucketCache(1487): Thread RS:1;86162e2766a8:38283 started persisting bucket cache to file 2024-12-05T19:45:24,546 INFO [RS:2;86162e2766a8:39853 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T19:45:24,546 DEBUG [RS:1;86162e2766a8:38283 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:24,546 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T19:45:24,546 DEBUG [RS:1;86162e2766a8:38283 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:24,546 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T19:45:24,546 INFO [RS:2;86162e2766a8:39853 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0572144-8026-ce2d-6e93-dd727cf14d1c/bucket.persistence 2024-12-05T19:45:24,546 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T19:45:24,546 INFO [RS:2;86162e2766a8:39853 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T19:45:24,546 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T19:45:24,550 DEBUG [RS:1;86162e2766a8:38283 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T19:45:24,550 DEBUG [RS:1;86162e2766a8:38283 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733427924546 2024-12-05T19:45:24,550 DEBUG [RS:1;86162e2766a8:38283 {}] bucket.BucketCache(1509): Thread RS:1;86162e2766a8:38283 finished persisting bucket cache to file, renaming 2024-12-05T19:45:24,550 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T19:45:24,551 INFO [RS:1;86162e2766a8:38283 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T19:45:24,551 INFO [RS:1;86162e2766a8:38283 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T19:45:24,551 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T19:45:24,551 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(3091): Received CLOSE for 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:24,551 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(959): stopping server 86162e2766a8,38283,1733427920690 2024-12-05T19:45:24,551 INFO [RS:1;86162e2766a8:38283 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T19:45:24,551 INFO [RS:1;86162e2766a8:38283 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;86162e2766a8:38283. 2024-12-05T19:45:24,551 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T19:45:24,551 DEBUG [RS:1;86162e2766a8:38283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T19:45:24,551 DEBUG [RS:1;86162e2766a8:38283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:24,551 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3ef2e4eb9a86b2ef6359f7a33978fa77, disabling compactions & flushes 2024-12-05T19:45:24,551 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:24,551 INFO [RS:1;86162e2766a8:38283 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T19:45:24,551 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:24,551 INFO [RS:1;86162e2766a8:38283 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T19:45:24,552 INFO [RS:1;86162e2766a8:38283 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T19:45:24,552 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. after waiting 0 ms 2024-12-05T19:45:24,552 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T19:45:24,552 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:24,552 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-05T19:45:24,552 DEBUG [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(1325): Online Regions={3ef2e4eb9a86b2ef6359f7a33978fa77=testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77., 1588230740=hbase:meta,,1.1588230740} 2024-12-05T19:45:24,552 DEBUG [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3ef2e4eb9a86b2ef6359f7a33978fa77 2024-12-05T19:45:24,552 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T19:45:24,552 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T19:45:24,552 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T19:45:24,552 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T19:45:24,552 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T19:45:24,552 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T19:45:24,552 DEBUG [StoreFileCloser-3ef2e4eb9a86b2ef6359f7a33978fa77-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/family/09d3acbbd7c0483c842346f25dc80c43 2024-12-05T19:45:24,553 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-05T19:45:24,556 DEBUG [RS:2;86162e2766a8:39853 {}] bucket.BucketCache(1487): Thread RS:2;86162e2766a8:39853 started persisting bucket cache to file 2024-12-05T19:45:24,557 DEBUG [RS:2;86162e2766a8:39853 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T19:45:24,557 DEBUG [RS:2;86162e2766a8:39853 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T19:45:24,558 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/default/testBlockEvictionOnGracefulStop1733427921732/3ef2e4eb9a86b2ef6359f7a33978fa77/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-05T19:45:24,558 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:24,558 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3ef2e4eb9a86b2ef6359f7a33978fa77: Waiting for close lock at 1733427924551Running coprocessor pre-close hooks at 1733427924551Disabling compacts and flushes for region at 1733427924551Disabling writes for close at 1733427924552 (+1 ms)Writing region close event to WAL at 1733427924553 (+1 ms)Running coprocessor post-close hooks at 1733427924558 (+5 ms)Closed at 1733427924558 2024-12-05T19:45:24,558 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77. 2024-12-05T19:45:24,561 DEBUG [RS:2;86162e2766a8:39853 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T19:45:24,561 DEBUG [RS:2;86162e2766a8:39853 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733427924557 2024-12-05T19:45:24,561 DEBUG [RS:2;86162e2766a8:39853 {}] bucket.BucketCache(1509): Thread RS:2;86162e2766a8:39853 finished persisting bucket cache to file, renaming 2024-12-05T19:45:24,561 INFO [RS:2;86162e2766a8:39853 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T19:45:24,561 INFO [RS:2;86162e2766a8:39853 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T19:45:24,561 INFO [RS:2;86162e2766a8:39853 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T19:45:24,561 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T19:45:24,561 INFO [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(959): stopping server 86162e2766a8,39853,1733427923945 2024-12-05T19:45:24,561 INFO [RS:2;86162e2766a8:39853 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T19:45:24,561 INFO [RS:2;86162e2766a8:39853 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;86162e2766a8:39853. 2024-12-05T19:45:24,561 DEBUG [RS:2;86162e2766a8:39853 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T19:45:24,561 DEBUG [RS:2;86162e2766a8:39853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:24,561 INFO [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(976): stopping server 86162e2766a8,39853,1733427923945; all regions closed. 2024-12-05T19:45:24,563 DEBUG [RS:2;86162e2766a8:39853 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs 2024-12-05T19:45:24,563 INFO [RS:2;86162e2766a8:39853 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 86162e2766a8%2C39853%2C1733427923945:(num 1733427924145) 2024-12-05T19:45:24,563 DEBUG [RS:2;86162e2766a8:39853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:24,563 INFO [RS:2;86162e2766a8:39853 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:24,564 INFO [RS:2;86162e2766a8:39853 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T19:45:24,564 INFO [RS:2;86162e2766a8:39853 {}] hbase.ChoreService(370): Chore service for: regionserver/86162e2766a8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T19:45:24,564 INFO [RS:2;86162e2766a8:39853 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T19:45:24,564 INFO [RS:2;86162e2766a8:39853 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T19:45:24,564 INFO [regionserver/86162e2766a8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T19:45:24,564 INFO [RS:2;86162e2766a8:39853 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T19:45:24,564 INFO [RS:2;86162e2766a8:39853 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T19:45:24,564 INFO [RS:2;86162e2766a8:39853 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39853 2024-12-05T19:45:24,564 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ca784b83d28245f886d58c510ceaf62e 2024-12-05T19:45:24,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T19:45:24,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/86162e2766a8,39853,1733427923945 2024-12-05T19:45:24,567 INFO [RS:2;86162e2766a8:39853 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T19:45:24,569 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [86162e2766a8,39853,1733427923945] 2024-12-05T19:45:24,571 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/86162e2766a8,39853,1733427923945 already deleted, retry=false 2024-12-05T19:45:24,571 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 86162e2766a8,39853,1733427923945 expired; onlineServers=1 2024-12-05T19:45:24,577 INFO [regionserver/86162e2766a8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:24,580 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/info/ca784b83d28245f886d58c510ceaf62e is 209, key is testBlockEvictionOnGracefulStop1733427921732,,1733427921737.3ef2e4eb9a86b2ef6359f7a33978fa77./info:regioninfo/1733427923995/Put/seqid=0 2024-12-05T19:45:24,580 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/info/ca784b83d28245f886d58c510ceaf62e 2024-12-05T19:45:24,589 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 911ac2b6fbd9411ead7468d0931ed7d9 2024-12-05T19:45:24,602 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/ns/911ac2b6fbd9411ead7468d0931ed7d9 is 43, key is default/ns:d/1733427921639/Put/seqid=0 2024-12-05T19:45:24,602 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/ns/911ac2b6fbd9411ead7468d0931ed7d9 2024-12-05T19:45:24,608 INFO [regionserver/86162e2766a8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:24,611 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f6b34440de034e86949c3f522dd3b7e7 2024-12-05T19:45:24,631 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/table/f6b34440de034e86949c3f522dd3b7e7 is 80, key is testBlockEvictionOnGracefulStop1733427921732/table:state/1733427922117/Put/seqid=0 2024-12-05T19:45:24,632 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/table/f6b34440de034e86949c3f522dd3b7e7 2024-12-05T19:45:24,635 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/info/ca784b83d28245f886d58c510ceaf62e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/info/ca784b83d28245f886d58c510ceaf62e 2024-12-05T19:45:24,637 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/info/ca784b83d28245f886d58c510ceaf62e, delay=931 ms 2024-12-05T19:45:24,638 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/info/ca784b83d28245f886d58c510ceaf62e, entries=14, sequenceid=13, filesize=7.5 K 2024-12-05T19:45:24,639 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/ns/911ac2b6fbd9411ead7468d0931ed7d9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/ns/911ac2b6fbd9411ead7468d0931ed7d9 2024-12-05T19:45:24,640 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/ns/911ac2b6fbd9411ead7468d0931ed7d9, delay=951 ms 2024-12-05T19:45:24,642 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/ns/911ac2b6fbd9411ead7468d0931ed7d9, entries=2, sequenceid=13, filesize=5.0 K 2024-12-05T19:45:24,642 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/.tmp/table/f6b34440de034e86949c3f522dd3b7e7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/table/f6b34440de034e86949c3f522dd3b7e7 2024-12-05T19:45:24,644 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/table/f6b34440de034e86949c3f522dd3b7e7, delay=989 ms 2024-12-05T19:45:24,645 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/table/f6b34440de034e86949c3f522dd3b7e7, entries=2, sequenceid=13, filesize=5.3 K 2024-12-05T19:45:24,646 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=13, compaction requested=false 2024-12-05T19:45:24,647 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/info/ca784b83d28245f886d58c510ceaf62e 2024-12-05T19:45:24,647 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/ns/911ac2b6fbd9411ead7468d0931ed7d9 2024-12-05T19:45:24,648 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/table/f6b34440de034e86949c3f522dd3b7e7 2024-12-05T19:45:24,653 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-05T19:45:24,653 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T19:45:24,653 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T19:45:24,653 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733427924552Running coprocessor pre-close hooks at 1733427924552Disabling compacts and flushes for region at 1733427924552Disabling writes for close at 1733427924552Obtaining lock to block concurrent updates at 1733427924553 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733427924553Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733427924553Flushing stores of hbase:meta,,1.1588230740 at 1733427924554 (+1 ms)Flushing 1588230740/info: creating writer at 1733427924554Flushing 1588230740/info: appending metadata at 1733427924579 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733427924579Flushing 1588230740/ns: creating writer at 1733427924583 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733427924601 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733427924601Flushing 1588230740/table: creating writer at 1733427924604 (+3 ms)Flushing 1588230740/table: appending metadata at 1733427924630 (+26 ms)Flushing 1588230740/table: closing flushed file at 1733427924630Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e5ee2a: reopening flushed file at 1733427924635 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b23d6f9: reopening flushed file at 1733427924639 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70499730: reopening flushed file at 1733427924642 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=13, compaction requested=false at 1733427924646 (+4 ms)Writing region close event to WAL at 1733427924648 (+2 ms)Running coprocessor post-close hooks at 1733427924653 (+5 ms)Closed at 1733427924653 2024-12-05T19:45:24,653 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T19:45:24,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:24,669 INFO [RS:2;86162e2766a8:39853 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T19:45:24,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39853-0x10063b6b7b10003, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:24,669 INFO [RS:2;86162e2766a8:39853 {}] regionserver.HRegionServer(1031): Exiting; stopping=86162e2766a8,39853,1733427923945; zookeeper connection closed. 2024-12-05T19:45:24,670 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d324d8f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d324d8f 2024-12-05T19:45:24,752 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(976): stopping server 86162e2766a8,38283,1733427920690; all regions closed. 2024-12-05T19:45:24,754 DEBUG [RS:1;86162e2766a8:38283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs 2024-12-05T19:45:24,755 INFO [RS:1;86162e2766a8:38283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 86162e2766a8%2C38283%2C1733427920690.meta:.meta(num 1733427921587) 2024-12-05T19:45:24,756 DEBUG [RS:1;86162e2766a8:38283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/oldWALs 2024-12-05T19:45:24,756 INFO [RS:1;86162e2766a8:38283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 86162e2766a8%2C38283%2C1733427920690:(num 1733427921212) 2024-12-05T19:45:24,756 DEBUG [RS:1;86162e2766a8:38283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T19:45:24,756 INFO [RS:1;86162e2766a8:38283 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T19:45:24,756 INFO [RS:1;86162e2766a8:38283 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T19:45:24,757 INFO [RS:1;86162e2766a8:38283 {}] hbase.ChoreService(370): Chore service for: regionserver/86162e2766a8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T19:45:24,757 INFO [RS:1;86162e2766a8:38283 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T19:45:24,757 INFO [regionserver/86162e2766a8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T19:45:24,757 INFO [RS:1;86162e2766a8:38283 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38283 2024-12-05T19:45:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T19:45:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/86162e2766a8,38283,1733427920690 2024-12-05T19:45:24,759 INFO [RS:1;86162e2766a8:38283 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T19:45:24,761 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [86162e2766a8,38283,1733427920690] 2024-12-05T19:45:24,762 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/86162e2766a8,38283,1733427920690 already deleted, retry=false 2024-12-05T19:45:24,762 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 86162e2766a8,38283,1733427920690 expired; onlineServers=0 2024-12-05T19:45:24,762 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '86162e2766a8,36249,1733427920604' ***** 2024-12-05T19:45:24,762 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T19:45:24,762 INFO [M:0;86162e2766a8:36249 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T19:45:24,762 INFO [M:0;86162e2766a8:36249 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T19:45:24,763 DEBUG [M:0;86162e2766a8:36249 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T19:45:24,763 DEBUG [M:0;86162e2766a8:36249 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T19:45:24,763 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T19:45:24,763 DEBUG [master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.large.0-1733427920952 {}] cleaner.HFileCleaner(306): Exit Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.large.0-1733427920952,5,FailOnTimeoutGroup] 2024-12-05T19:45:24,763 INFO [M:0;86162e2766a8:36249 {}] hbase.ChoreService(370): Chore service for: master/86162e2766a8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T19:45:24,763 DEBUG [master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.small.0-1733427920956 {}] cleaner.HFileCleaner(306): Exit Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.small.0-1733427920956,5,FailOnTimeoutGroup] 2024-12-05T19:45:24,763 INFO [M:0;86162e2766a8:36249 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T19:45:24,763 DEBUG [M:0;86162e2766a8:36249 {}] master.HMaster(1795): Stopping service threads 2024-12-05T19:45:24,763 INFO [M:0;86162e2766a8:36249 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T19:45:24,763 INFO [M:0;86162e2766a8:36249 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T19:45:24,763 INFO [M:0;86162e2766a8:36249 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T19:45:24,763 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T19:45:24,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T19:45:24,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T19:45:24,765 DEBUG [M:0;86162e2766a8:36249 {}] zookeeper.ZKUtil(347): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T19:45:24,765 WARN [M:0;86162e2766a8:36249 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T19:45:24,766 INFO [M:0;86162e2766a8:36249 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/.lastflushedseqids 2024-12-05T19:45:24,772 INFO [M:0;86162e2766a8:36249 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T19:45:24,773 INFO [M:0;86162e2766a8:36249 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T19:45:24,773 DEBUG [M:0;86162e2766a8:36249 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T19:45:24,773 INFO [M:0;86162e2766a8:36249 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:24,773 DEBUG [M:0;86162e2766a8:36249 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:24,773 DEBUG [M:0;86162e2766a8:36249 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T19:45:24,773 DEBUG [M:0;86162e2766a8:36249 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:24,773 INFO [M:0;86162e2766a8:36249 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-12-05T19:45:24,784 DEBUG [M:0;86162e2766a8:36249 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 10fd6cca67a14854a8ff3a061a211095 2024-12-05T19:45:24,799 DEBUG [M:0;86162e2766a8:36249 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10fd6cca67a14854a8ff3a061a211095 is 82, key is hbase:meta,,1/info:regioninfo/1733427921622/Put/seqid=0 2024-12-05T19:45:24,799 INFO [M:0;86162e2766a8:36249 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10fd6cca67a14854a8ff3a061a211095 2024-12-05T19:45:24,807 DEBUG [M:0;86162e2766a8:36249 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b455882c65a54c4280dc9fdea7d0f86e 2024-12-05T19:45:24,822 DEBUG [M:0;86162e2766a8:36249 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b455882c65a54c4280dc9fdea7d0f86e is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733427922122/Put/seqid=0 2024-12-05T19:45:24,823 INFO [M:0;86162e2766a8:36249 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b455882c65a54c4280dc9fdea7d0f86e 2024-12-05T19:45:24,831 DEBUG [M:0;86162e2766a8:36249 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 57acc4f47b594f08bfd3b25aa57458d2 2024-12-05T19:45:24,846 DEBUG [M:0;86162e2766a8:36249 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/57acc4f47b594f08bfd3b25aa57458d2 is 69, key is 86162e2766a8,38283,1733427920690/rs:state/1733427921052/Put/seqid=0 2024-12-05T19:45:24,846 INFO [M:0;86162e2766a8:36249 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/57acc4f47b594f08bfd3b25aa57458d2 2024-12-05T19:45:24,848 INFO [M:0;86162e2766a8:36249 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 57acc4f47b594f08bfd3b25aa57458d2 2024-12-05T19:45:24,849 DEBUG [M:0;86162e2766a8:36249 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10fd6cca67a14854a8ff3a061a211095 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/10fd6cca67a14854a8ff3a061a211095 2024-12-05T19:45:24,851 INFO [M:0;86162e2766a8:36249 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/10fd6cca67a14854a8ff3a061a211095, entries=8, sequenceid=83, filesize=5.5 K 2024-12-05T19:45:24,851 DEBUG [M:0;86162e2766a8:36249 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b455882c65a54c4280dc9fdea7d0f86e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b455882c65a54c4280dc9fdea7d0f86e 2024-12-05T19:45:24,853 INFO [M:0;86162e2766a8:36249 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b455882c65a54c4280dc9fdea7d0f86e, entries=9, sequenceid=83, filesize=6.4 K 2024-12-05T19:45:24,854 DEBUG [M:0;86162e2766a8:36249 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/57acc4f47b594f08bfd3b25aa57458d2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/57acc4f47b594f08bfd3b25aa57458d2 2024-12-05T19:45:24,856 INFO [M:0;86162e2766a8:36249 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 57acc4f47b594f08bfd3b25aa57458d2 2024-12-05T19:45:24,856 INFO [M:0;86162e2766a8:36249 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/30a62873-edb6-592a-0817-2d0fa04b175e/5d45ca5a-1dfb-dbd4-351f-41d0d856d3d7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/57acc4f47b594f08bfd3b25aa57458d2, entries=3, sequenceid=83, filesize=5.3 K 2024-12-05T19:45:24,857 INFO [M:0;86162e2766a8:36249 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=83, compaction requested=false 2024-12-05T19:45:24,858 INFO [M:0;86162e2766a8:36249 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T19:45:24,859 DEBUG [M:0;86162e2766a8:36249 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733427924773Disabling compacts and flushes for region at 1733427924773Disabling writes for close at 1733427924773Obtaining lock to block concurrent updates at 1733427924773Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733427924773Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1733427924774 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733427924774Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733427924774Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733427924798 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733427924798Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733427924801 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733427924822 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733427924822Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733427924825 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733427924845 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733427924845Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@451f6a6e: reopening flushed file at 1733427924848 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36638b04: reopening flushed file at 1733427924851 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2304f1bf: reopening flushed file at 1733427924853 (+2 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=83, compaction requested=false at 1733427924857 (+4 ms)Writing region close event to WAL at 1733427924858 (+1 ms)Closed at 1733427924858 2024-12-05T19:45:24,860 INFO [M:0;86162e2766a8:36249 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T19:45:24,860 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T19:45:24,860 INFO [M:0;86162e2766a8:36249 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36249 2024-12-05T19:45:24,860 INFO [M:0;86162e2766a8:36249 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T19:45:24,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:24,861 INFO [RS:1;86162e2766a8:38283 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T19:45:24,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38283-0x10063b6b7b10002, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:24,861 INFO [RS:1;86162e2766a8:38283 {}] regionserver.HRegionServer(1031): Exiting; stopping=86162e2766a8,38283,1733427920690; zookeeper connection closed. 2024-12-05T19:45:24,861 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@23abc2b8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@23abc2b8 2024-12-05T19:45:24,862 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-05T19:45:24,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:24,963 INFO [M:0;86162e2766a8:36249 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T19:45:24,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36249-0x10063b6b7b10000, quorum=127.0.0.1:50580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T19:45:24,966 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T19:45:24,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-05T19:45:24,967 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T19:45:24,974 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 681), ProcessCount=11 (was 11), AvailableMemoryMB=8926 (was 9031)