2024-12-07 18:01:53,526 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-07 18:01:53,544 main DEBUG Took 0.009810 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 18:01:53,544 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 18:01:53,544 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 18:01:53,545 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 18:01:53,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,552 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 18:01:53,561 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,563 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,564 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,565 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,565 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,566 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,567 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,567 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,567 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,568 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,568 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,568 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,569 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,569 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,569 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,569 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,570 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:01:53,570 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,571 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 18:01:53,573 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:01:53,574 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 18:01:53,576 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 18:01:53,576 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 18:01:53,578 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 18:01:53,578 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 18:01:53,586 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 18:01:53,588 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 18:01:53,589 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 18:01:53,590 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 18:01:53,590 main DEBUG createAppenders(={Console}) 2024-12-07 18:01:53,591 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-07 18:01:53,591 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-07 18:01:53,592 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-07 18:01:53,592 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 18:01:53,592 main DEBUG OutputStream closed 2024-12-07 18:01:53,593 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 18:01:53,593 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 18:01:53,593 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-07 18:01:53,671 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 18:01:53,673 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 18:01:53,674 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 18:01:53,675 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 18:01:53,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 18:01:53,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 18:01:53,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 18:01:53,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 18:01:53,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 18:01:53,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 18:01:53,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 18:01:53,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 18:01:53,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 18:01:53,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 18:01:53,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 18:01:53,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 18:01:53,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 18:01:53,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 18:01:53,683 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 18:01:53,684 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-07 18:01:53,684 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 18:01:53,685 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-07T18:01:53,699 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-07 18:01:53,704 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 18:01:53,704 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T18:01:53,929 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda 2024-12-07T18:01:53,952 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=20956 2024-12-07T18:01:54,052 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T18:01:54,129 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:01:54,145 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/cluster_7aad5014-2fa2-7bc9-a0d4-aa5ed75b626f, deleteOnExit=true 2024-12-07T18:01:54,276 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/cluster_7aad5014-2fa2-7bc9-a0d4-aa5ed75b626f/zookeeper_0, clientPort=56707, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/cluster_7aad5014-2fa2-7bc9-a0d4-aa5ed75b626f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/cluster_7aad5014-2fa2-7bc9-a0d4-aa5ed75b626f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T18:01:54,288 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56707 2024-12-07T18:01:54,289 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:01:54,362 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 with version=8 2024-12-07T18:01:54,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/hbase-staging 2024-12-07T18:01:54,463 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-07T18:01:54,772 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bd53b59592b3:0 server-side Connection retries=45 2024-12-07T18:01:54,781 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:01:54,782 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T18:01:54,789 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T18:01:54,790 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:01:54,790 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T18:01:54,909 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T18:01:54,999 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T18:01:55,039 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T18:01:55,049 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T18:01:55,052 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T18:01:55,087 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 110237 (auto-detected) 2024-12-07T18:01:55,088 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-07T18:01:55,109 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34575 2024-12-07T18:01:55,136 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34575 connecting to ZooKeeper ensemble=127.0.0.1:56707 2024-12-07T18:01:55,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:345750x0, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T18:01:55,263 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34575-0x100017ec1ea0000 connected 2024-12-07T18:01:55,354 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:01:55,355 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:01:55,366 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:01:55,370 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9, hbase.cluster.distributed=false 2024-12-07T18:01:55,395 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T18:01:55,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34575 2024-12-07T18:01:55,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34575 2024-12-07T18:01:55,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34575 2024-12-07T18:01:55,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34575 2024-12-07T18:01:55,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34575 2024-12-07T18:01:55,478 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bd53b59592b3:0 server-side Connection retries=45 2024-12-07T18:01:55,480 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:01:55,480 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T18:01:55,480 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T18:01:55,480 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:01:55,481 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T18:01:55,484 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T18:01:55,487 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T18:01:55,488 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42075 2024-12-07T18:01:55,490 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42075 connecting to ZooKeeper ensemble=127.0.0.1:56707 2024-12-07T18:01:55,490 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:01:55,491 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:01:55,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420750x0, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T18:01:55,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:420750x0, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:01:55,509 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42075-0x100017ec1ea0001 connected 2024-12-07T18:01:55,515 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T18:01:55,535 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.cache 2024-12-07T18:01:55,542 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T18:01:55,544 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T18:01:55,546 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T18:01:55,551 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T18:01:55,551 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T18:01:55,561 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T18:01:55,561 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.persistence is complete. 2024-12-07T18:01:55,562 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T18:01:55,570 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T18:01:55,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T18:01:55,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T18:01:55,586 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42075 2024-12-07T18:01:55,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42075 2024-12-07T18:01:55,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42075 2024-12-07T18:01:55,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42075 2024-12-07T18:01:55,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42075 2024-12-07T18:01:55,617 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bd53b59592b3:0 server-side Connection retries=45 2024-12-07T18:01:55,617 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:01:55,617 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T18:01:55,617 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T18:01:55,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:01:55,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T18:01:55,618 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T18:01:55,618 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T18:01:55,620 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39593 2024-12-07T18:01:55,623 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39593 connecting to ZooKeeper ensemble=127.0.0.1:56707 2024-12-07T18:01:55,623 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:01:55,624 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:01:55,650 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:395930x0, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T18:01:55,650 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:395930x0, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:01:55,651 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T18:01:55,652 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39593-0x100017ec1ea0002 connected 2024-12-07T18:01:55,652 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.cache 2024-12-07T18:01:55,653 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T18:01:55,653 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T18:01:55,653 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T18:01:55,654 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T18:01:55,655 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T18:01:55,655 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T18:01:55,655 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T18:01:55,655 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.persistence is complete. 2024-12-07T18:01:55,659 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T18:01:55,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T18:01:55,663 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T18:01:55,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39593 2024-12-07T18:01:55,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39593 2024-12-07T18:01:55,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39593 2024-12-07T18:01:55,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39593 2024-12-07T18:01:55,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39593 2024-12-07T18:01:55,684 DEBUG [M:0;bd53b59592b3:34575 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bd53b59592b3:34575 2024-12-07T18:01:55,685 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bd53b59592b3,34575,1733594514554 2024-12-07T18:01:55,700 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:01:55,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:01:55,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:01:55,702 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bd53b59592b3,34575,1733594514554 2024-12-07T18:01:55,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T18:01:55,733 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T18:01:55,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:55,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:55,733 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:55,735 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T18:01:55,737 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bd53b59592b3,34575,1733594514554 from backup master directory 2024-12-07T18:01:55,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:01:55,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bd53b59592b3,34575,1733594514554 2024-12-07T18:01:55,749 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:01:55,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:01:55,750 WARN [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T18:01:55,751 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bd53b59592b3,34575,1733594514554 2024-12-07T18:01:55,753 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-07T18:01:55,754 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-07T18:01:55,761 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/hbase.id] with ID: ea98f292-fb08-40e7-909c-26c008c1beab 2024-12-07T18:01:55,761 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/.tmp/hbase.id 2024-12-07T18:01:55,766 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/hbase.id] 2024-12-07T18:01:55,797 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:01:55,801 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T18:01:55,818 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-07T18:01:55,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:55,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:55,833 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:55,862 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:01:55,865 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T18:01:55,904 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:01:55,909 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:01:55,978 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store 2024-12-07T18:01:56,005 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T18:01:56,010 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:01:56,012 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T18:01:56,012 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:01:56,013 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:01:56,015 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T18:01:56,015 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:01:56,016 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:01:56,016 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733594516012Disabling compacts and flushes for region at 1733594516012Disabling writes for close at 1733594516015 (+3 ms)Writing region close event to WAL at 1733594516015Closed at 1733594516015 2024-12-07T18:01:56,017 WARN [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/.initializing 2024-12-07T18:01:56,018 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/WALs/bd53b59592b3,34575,1733594514554 2024-12-07T18:01:56,030 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T18:01:56,043 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd53b59592b3%2C34575%2C1733594514554, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/WALs/bd53b59592b3,34575,1733594514554, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/oldWALs, maxLogs=10 2024-12-07T18:01:56,079 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/WALs/bd53b59592b3,34575,1733594514554/bd53b59592b3%2C34575%2C1733594514554.1733594516047 2024-12-07T18:01:56,080 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T18:01:56,081 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:01:56,081 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:01:56,085 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,086 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T18:01:56,159 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:56,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:01:56,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,176 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T18:01:56,176 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:56,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:01:56,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,201 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T18:01:56,202 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:56,202 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:01:56,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,217 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T18:01:56,217 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:56,217 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:01:56,218 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,223 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,224 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,228 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,229 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,232 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T18:01:56,235 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:01:56,248 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:01:56,249 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69444295, jitterRate=0.034800633788108826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T18:01:56,257 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733594516099Initializing all the Stores at 1733594516099Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594516100 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594516100Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594516101 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594516101Cleaning up temporary data from old regions at 1733594516229 (+128 ms)Region opened successfully at 1733594516256 (+27 ms) 2024-12-07T18:01:56,258 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T18:01:56,300 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbbf432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd53b59592b3/172.17.0.2:0 2024-12-07T18:01:56,345 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T18:01:56,362 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T18:01:56,362 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T18:01:56,366 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T18:01:56,367 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T18:01:56,375 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-07T18:01:56,376 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T18:01:56,406 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T18:01:56,417 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T18:01:56,474 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T18:01:56,479 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T18:01:56,481 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T18:01:56,491 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T18:01:56,494 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T18:01:56,500 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T18:01:56,507 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T18:01:56,509 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T18:01:56,515 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T18:01:56,537 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T18:01:56,549 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T18:01:56,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T18:01:56,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T18:01:56,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:56,573 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T18:01:56,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:56,573 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:56,576 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bd53b59592b3,34575,1733594514554, sessionid=0x100017ec1ea0000, setting cluster-up flag (Was=false) 2024-12-07T18:01:56,608 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:56,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:56,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:56,633 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T18:01:56,635 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bd53b59592b3,34575,1733594514554 2024-12-07T18:01:56,657 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:56,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:56,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:56,683 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T18:01:56,685 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bd53b59592b3,34575,1733594514554 2024-12-07T18:01:56,692 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T18:01:56,775 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(746): ClusterId : ea98f292-fb08-40e7-909c-26c008c1beab 2024-12-07T18:01:56,775 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(746): ClusterId : ea98f292-fb08-40e7-909c-26c008c1beab 2024-12-07T18:01:56,778 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T18:01:56,778 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T18:01:56,781 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T18:01:56,795 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T18:01:56,803 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T18:01:56,803 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T18:01:56,803 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T18:01:56,803 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T18:01:56,805 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T18:01:56,813 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bd53b59592b3,34575,1733594514554 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T18:01:56,825 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T18:01:56,825 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T18:01:56,826 DEBUG [RS:0;bd53b59592b3:42075 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cba953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd53b59592b3/172.17.0.2:0 2024-12-07T18:01:56,826 DEBUG [RS:1;bd53b59592b3:39593 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ec6a19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd53b59592b3/172.17.0.2:0 2024-12-07T18:01:56,827 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bd53b59592b3:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:01:56,828 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bd53b59592b3:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:01:56,828 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bd53b59592b3:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:01:56,828 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bd53b59592b3:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:01:56,828 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bd53b59592b3:0, corePoolSize=10, maxPoolSize=10 2024-12-07T18:01:56,829 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:56,829 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bd53b59592b3:0, corePoolSize=2, maxPoolSize=2 2024-12-07T18:01:56,829 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:56,843 DEBUG [RS:1;bd53b59592b3:39593 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;bd53b59592b3:39593 2024-12-07T18:01:56,843 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T18:01:56,844 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T18:01:56,848 DEBUG [RS:0;bd53b59592b3:42075 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bd53b59592b3:42075 2024-12-07T18:01:56,848 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T18:01:56,848 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:56,848 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T18:01:56,848 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T18:01:56,848 DEBUG [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T18:01:56,848 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T18:01:56,848 DEBUG [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T18:01:56,848 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T18:01:56,851 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd53b59592b3,34575,1733594514554 with port=42075, startcode=1733594515444 2024-12-07T18:01:56,851 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd53b59592b3,34575,1733594514554 with port=39593, startcode=1733594515616 2024-12-07T18:01:56,856 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733594546856 2024-12-07T18:01:56,858 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T18:01:56,860 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T18:01:56,864 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T18:01:56,864 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T18:01:56,865 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T18:01:56,865 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T18:01:56,866 DEBUG [RS:1;bd53b59592b3:39593 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T18:01:56,866 DEBUG [RS:0;bd53b59592b3:42075 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T18:01:56,867 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:56,877 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T18:01:56,878 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T18:01:56,879 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T18:01:56,882 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T18:01:56,882 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T18:01:56,884 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.large.0-1733594516883,5,FailOnTimeoutGroup] 2024-12-07T18:01:56,885 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.small.0-1733594516884,5,FailOnTimeoutGroup] 2024-12-07T18:01:56,885 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:56,886 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T18:01:56,886 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T18:01:56,887 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:56,887 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 2024-12-07T18:01:56,887 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:56,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:01:56,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T18:01:56,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T18:01:56,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:56,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:01:56,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T18:01:56,919 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T18:01:56,919 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:56,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:01:56,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T18:01:56,931 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T18:01:56,931 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:56,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:01:56,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T18:01:56,946 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T18:01:56,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:56,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:01:56,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T18:01:56,947 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53463, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T18:01:56,947 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52795, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T18:01:56,948 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740 2024-12-07T18:01:56,948 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740 2024-12-07T18:01:56,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T18:01:56,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T18:01:56,949 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T18:01:56,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T18:01:56,955 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34575 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bd53b59592b3,39593,1733594515616 2024-12-07T18:01:56,958 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34575 {}] master.ServerManager(517): Registering regionserver=bd53b59592b3,39593,1733594515616 2024-12-07T18:01:56,963 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:01:56,964 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66273756, jitterRate=-0.012444078922271729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T18:01:56,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733594516901Initializing all the Stores at 1733594516902 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594516902Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594516902Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594516902Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594516902Cleaning up temporary data from old regions at 1733594516949 (+47 ms)Region opened successfully at 1733594516968 (+19 ms) 2024-12-07T18:01:56,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T18:01:56,968 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T18:01:56,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T18:01:56,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T18:01:56,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T18:01:56,971 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T18:01:56,971 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733594516968Disabling compacts and flushes for region at 1733594516968Disabling writes for close at 1733594516969 (+1 ms)Writing region close event to WAL at 1733594516970 (+1 ms)Closed at 1733594516970 2024-12-07T18:01:56,972 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34575 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bd53b59592b3,42075,1733594515444 2024-12-07T18:01:56,973 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34575 {}] master.ServerManager(517): Registering regionserver=bd53b59592b3,42075,1733594515444 2024-12-07T18:01:56,974 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T18:01:56,974 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T18:01:56,977 DEBUG [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 2024-12-07T18:01:56,978 DEBUG [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T18:01:56,978 DEBUG [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T18:01:56,979 DEBUG [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 2024-12-07T18:01:56,979 DEBUG [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T18:01:56,979 DEBUG [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T18:01:56,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T18:01:56,997 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T18:01:57,000 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T18:01:57,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:01:57,035 DEBUG [RS:0;bd53b59592b3:42075 {}] zookeeper.ZKUtil(111): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bd53b59592b3,42075,1733594515444 2024-12-07T18:01:57,035 WARN [RS:0;bd53b59592b3:42075 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T18:01:57,035 INFO [RS:0;bd53b59592b3:42075 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:01:57,035 DEBUG [RS:1;bd53b59592b3:39593 {}] zookeeper.ZKUtil(111): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bd53b59592b3,39593,1733594515616 2024-12-07T18:01:57,036 WARN [RS:1;bd53b59592b3:39593 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T18:01:57,036 DEBUG [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42075,1733594515444 2024-12-07T18:01:57,036 INFO [RS:1;bd53b59592b3:39593 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:01:57,036 DEBUG [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,39593,1733594515616 2024-12-07T18:01:57,037 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bd53b59592b3,39593,1733594515616] 2024-12-07T18:01:57,037 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bd53b59592b3,42075,1733594515444] 2024-12-07T18:01:57,070 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T18:01:57,070 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T18:01:57,088 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T18:01:57,088 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T18:01:57,093 INFO [RS:0;bd53b59592b3:42075 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T18:01:57,093 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,094 INFO [RS:1;bd53b59592b3:39593 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T18:01:57,094 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,099 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T18:01:57,099 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T18:01:57,105 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T18:01:57,105 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T18:01:57,107 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,107 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,108 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,108 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,108 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,108 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,108 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,108 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,108 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,108 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,108 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,108 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bd53b59592b3:0, corePoolSize=2, maxPoolSize=2 2024-12-07T18:01:57,109 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bd53b59592b3:0, corePoolSize=2, maxPoolSize=2 2024-12-07T18:01:57,109 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,109 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,110 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,110 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:01:57,110 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:01:57,110 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:01:57,110 DEBUG [RS:0;bd53b59592b3:42075 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:01:57,110 DEBUG [RS:1;bd53b59592b3:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:01:57,112 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,112 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,112 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,113 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,113 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,113 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,42075,1733594515444-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T18:01:57,115 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,115 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,115 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,115 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,115 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,115 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,39593,1733594515616-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T18:01:57,143 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T18:01:57,145 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,42075,1733594515444-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,146 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,146 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.Replication(171): bd53b59592b3,42075,1733594515444 started 2024-12-07T18:01:57,148 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T18:01:57,148 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,39593,1733594515616-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,149 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,149 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.Replication(171): bd53b59592b3,39593,1733594515616 started 2024-12-07T18:01:57,151 WARN [bd53b59592b3:34575 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T18:01:57,172 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,172 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,172 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(1482): Serving as bd53b59592b3,39593,1733594515616, RpcServer on bd53b59592b3/172.17.0.2:39593, sessionid=0x100017ec1ea0002 2024-12-07T18:01:57,172 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(1482): Serving as bd53b59592b3,42075,1733594515444, RpcServer on bd53b59592b3/172.17.0.2:42075, sessionid=0x100017ec1ea0001 2024-12-07T18:01:57,173 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T18:01:57,173 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T18:01:57,173 DEBUG [RS:1;bd53b59592b3:39593 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bd53b59592b3,39593,1733594515616 2024-12-07T18:01:57,173 DEBUG [RS:0;bd53b59592b3:42075 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bd53b59592b3,42075,1733594515444 2024-12-07T18:01:57,174 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,42075,1733594515444' 2024-12-07T18:01:57,174 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,39593,1733594515616' 2024-12-07T18:01:57,174 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T18:01:57,174 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T18:01:57,175 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T18:01:57,175 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T18:01:57,176 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T18:01:57,176 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T18:01:57,176 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T18:01:57,176 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T18:01:57,176 DEBUG [RS:1;bd53b59592b3:39593 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bd53b59592b3,39593,1733594515616 2024-12-07T18:01:57,176 DEBUG [RS:0;bd53b59592b3:42075 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bd53b59592b3,42075,1733594515444 2024-12-07T18:01:57,176 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,39593,1733594515616' 2024-12-07T18:01:57,176 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,42075,1733594515444' 2024-12-07T18:01:57,176 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T18:01:57,176 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T18:01:57,177 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T18:01:57,178 DEBUG [RS:1;bd53b59592b3:39593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T18:01:57,178 INFO [RS:1;bd53b59592b3:39593 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T18:01:57,178 INFO [RS:1;bd53b59592b3:39593 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T18:01:57,179 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T18:01:57,180 DEBUG [RS:0;bd53b59592b3:42075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T18:01:57,180 INFO [RS:0;bd53b59592b3:42075 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T18:01:57,180 INFO [RS:0;bd53b59592b3:42075 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T18:01:57,284 INFO [RS:1;bd53b59592b3:39593 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T18:01:57,284 INFO [RS:0;bd53b59592b3:42075 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T18:01:57,285 INFO [RS:1;bd53b59592b3:39593 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bd53b59592b3%2C39593%2C1733594515616, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,39593,1733594515616, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs, maxLogs=55 2024-12-07T18:01:57,285 INFO [RS:0;bd53b59592b3:42075 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bd53b59592b3%2C42075%2C1733594515444, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42075,1733594515444, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs, maxLogs=55 2024-12-07T18:01:57,313 INFO [RS:1;bd53b59592b3:39593 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,39593,1733594515616/bd53b59592b3%2C39593%2C1733594515616.1733594517288 2024-12-07T18:01:57,315 DEBUG [RS:1;bd53b59592b3:39593 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T18:01:57,315 INFO [RS:0;bd53b59592b3:42075 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42075,1733594515444/bd53b59592b3%2C42075%2C1733594515444.1733594517288 2024-12-07T18:01:57,316 DEBUG [RS:0;bd53b59592b3:42075 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T18:01:57,404 DEBUG [bd53b59592b3:34575 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T18:01:57,417 DEBUG [bd53b59592b3:34575 {}] balancer.BalancerClusterState(204): Hosts are {bd53b59592b3=0} racks are {/default-rack=0} 2024-12-07T18:01:57,421 DEBUG [bd53b59592b3:34575 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T18:01:57,421 DEBUG [bd53b59592b3:34575 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T18:01:57,421 DEBUG [bd53b59592b3:34575 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T18:01:57,421 DEBUG [bd53b59592b3:34575 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T18:01:57,421 INFO [bd53b59592b3:34575 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T18:01:57,421 INFO [bd53b59592b3:34575 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T18:01:57,422 DEBUG [bd53b59592b3:34575 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T18:01:57,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bd53b59592b3,42075,1733594515444 2024-12-07T18:01:57,434 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd53b59592b3,42075,1733594515444, state=OPENING 2024-12-07T18:01:57,490 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T18:01:57,499 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:57,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:57,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:01:57,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:01:57,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:01:57,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:01:57,503 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T18:01:57,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bd53b59592b3,42075,1733594515444}] 2024-12-07T18:01:57,682 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T18:01:57,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34847, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T18:01:57,696 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T18:01:57,696 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:01:57,697 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T18:01:57,699 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bd53b59592b3%2C42075%2C1733594515444.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42075,1733594515444, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs, maxLogs=55 2024-12-07T18:01:57,722 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42075,1733594515444/bd53b59592b3%2C42075%2C1733594515444.meta.1733594517701.meta 2024-12-07T18:01:57,723 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T18:01:57,723 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:01:57,725 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T18:01:57,728 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T18:01:57,734 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T18:01:57,739 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T18:01:57,739 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:01:57,740 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T18:01:57,740 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T18:01:57,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T18:01:57,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T18:01:57,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:57,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:01:57,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T18:01:57,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T18:01:57,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:57,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:01:57,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T18:01:57,746 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T18:01:57,747 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:57,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:01:57,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T18:01:57,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T18:01:57,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:57,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:01:57,750 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T18:01:57,751 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740 2024-12-07T18:01:57,752 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740 2024-12-07T18:01:57,752 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T18:01:57,752 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T18:01:57,753 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T18:01:57,754 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T18:01:57,754 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69913326, jitterRate=0.041789740324020386}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T18:01:57,755 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T18:01:57,756 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733594517741Writing region info on filesystem at 1733594517741Initializing all the Stores at 1733594517741Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594517741Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594517742 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594517742Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594517742Cleaning up temporary data from old regions at 1733594517752 (+10 ms)Running coprocessor post-open hooks at 1733594517755 (+3 ms)Region opened successfully at 1733594517756 (+1 ms) 2024-12-07T18:01:57,762 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733594517673 2024-12-07T18:01:57,774 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T18:01:57,774 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T18:01:57,775 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bd53b59592b3,42075,1733594515444 2024-12-07T18:01:57,777 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd53b59592b3,42075,1733594515444, state=OPEN 2024-12-07T18:01:57,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T18:01:57,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T18:01:57,806 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:01:57,806 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:01:57,807 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bd53b59592b3,42075,1733594515444 2024-12-07T18:01:57,807 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T18:01:57,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:01:57,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T18:01:57,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd53b59592b3,42075,1733594515444 in 302 msec 2024-12-07T18:01:57,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T18:01:57,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 829 msec 2024-12-07T18:01:57,818 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T18:01:57,818 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T18:01:57,840 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:01:57,841 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd53b59592b3,42075,1733594515444, seqNum=-1] 2024-12-07T18:01:57,865 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:01:57,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35059, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:01:57,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1610 sec 2024-12-07T18:01:57,891 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733594517891, completionTime=-1 2024-12-07T18:01:57,896 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-07T18:01:57,896 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T18:01:57,926 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T18:01:57,926 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733594577926 2024-12-07T18:01:57,926 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733594637926 2024-12-07T18:01:57,926 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-12-07T18:01:57,929 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-07T18:01:57,936 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,34575,1733594514554-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,936 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,34575,1733594514554-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,936 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,34575,1733594514554-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,938 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bd53b59592b3:34575, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,938 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,939 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,946 DEBUG [master/bd53b59592b3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T18:01:57,972 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.220sec 2024-12-07T18:01:57,974 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T18:01:57,975 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T18:01:57,977 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T18:01:57,977 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T18:01:57,977 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T18:01:57,978 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,34575,1733594514554-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T18:01:57,979 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,34575,1733594514554-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T18:01:57,983 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T18:01:57,984 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T18:01:57,984 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,34575,1733594514554-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:01:57,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f5010e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:01:57,991 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-07T18:01:57,992 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-07T18:01:57,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd53b59592b3,34575,-1 for getting cluster id 2024-12-07T18:01:57,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:01:58,009 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ea98f292-fb08-40e7-909c-26c008c1beab' 2024-12-07T18:01:58,013 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:01:58,013 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ea98f292-fb08-40e7-909c-26c008c1beab" 2024-12-07T18:01:58,016 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea769ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:01:58,016 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd53b59592b3,34575,-1] 2024-12-07T18:01:58,019 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:01:58,021 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:01:58,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:01:58,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea0dcfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:01:58,027 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:01:58,036 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd53b59592b3,42075,1733594515444, seqNum=-1] 2024-12-07T18:01:58,037 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:01:58,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:01:58,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bd53b59592b3,34575,1733594514554 2024-12-07T18:01:58,086 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T18:01:58,092 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bd53b59592b3,34575,1733594514554 2024-12-07T18:01:58,095 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b99b208 2024-12-07T18:01:58,096 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T18:01:58,098 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T18:01:58,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733594518078', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:01:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733594518078 2024-12-07T18:01:58,114 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733594518078 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T18:01:58,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733594518078" procId is: 4 2024-12-07T18:01:58,118 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:58,120 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733594518078 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T18:01:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:01:58,135 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733594518078-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 848ba8a19b86e6a65ace628f59502af6, NAME => 'testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733594518078', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 2024-12-07T18:01:58,142 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733594518078-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:01:58,143 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733594518078-pool-0 {}] regionserver.HRegion(1722): Closing 848ba8a19b86e6a65ace628f59502af6, disabling compactions & flushes 2024-12-07T18:01:58,143 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733594518078-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:58,143 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733594518078-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:58,143 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733594518078-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. after waiting 0 ms 2024-12-07T18:01:58,143 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733594518078-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:58,143 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733594518078-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:58,143 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733594518078-pool-0 {}] regionserver.HRegion(1676): Region close journal for 848ba8a19b86e6a65ace628f59502af6: Waiting for close lock at 1733594518142Disabling compacts and flushes for region at 1733594518142Disabling writes for close at 1733594518143 (+1 ms)Writing region close event to WAL at 1733594518143Closed at 1733594518143 2024-12-07T18:01:58,145 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733594518078 execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T18:01:58,151 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733594518145"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733594518145"}]},"ts":"1733594518145"} 2024-12-07T18:01:58,156 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T18:01:58,158 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733594518078 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T18:01:58,161 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733594518078","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733594518158"}]},"ts":"1733594518158"} 2024-12-07T18:01:58,166 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733594518078, state=ENABLING in hbase:meta 2024-12-07T18:01:58,167 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {bd53b59592b3=0} racks are {/default-rack=0} 2024-12-07T18:01:58,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T18:01:58,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T18:01:58,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T18:01:58,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T18:01:58,168 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T18:01:58,168 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T18:01:58,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T18:01:58,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, ASSIGN}] 2024-12-07T18:01:58,172 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, ASSIGN 2024-12-07T18:01:58,174 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, ASSIGN; state=OFFLINE, location=bd53b59592b3,39593,1733594515616; forceNewPlan=false, retain=false 2024-12-07T18:01:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:01:58,326 INFO [bd53b59592b3:34575 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T18:01:58,327 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=848ba8a19b86e6a65ace628f59502af6, regionState=OPENING, regionLocation=bd53b59592b3,39593,1733594515616 2024-12-07T18:01:58,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, ASSIGN because future has completed 2024-12-07T18:01:58,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 848ba8a19b86e6a65ace628f59502af6, server=bd53b59592b3,39593,1733594515616}] 2024-12-07T18:01:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:01:58,486 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T18:01:58,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49687, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T18:01:58,491 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:58,492 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 848ba8a19b86e6a65ace628f59502af6, NAME => 'testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:01:58,493 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733594518078 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,493 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:01:58,493 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,493 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,496 INFO [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,502 INFO [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 848ba8a19b86e6a65ace628f59502af6 columnFamilyName family 2024-12-07T18:01:58,502 DEBUG [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:58,503 INFO [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] regionserver.HStore(327): Store=848ba8a19b86e6a65ace628f59502af6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:01:58,506 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,507 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,507 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,508 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,508 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,509 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,516 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:01:58,516 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 848ba8a19b86e6a65ace628f59502af6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66118632, jitterRate=-0.014755606651306152}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T18:01:58,517 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:58,518 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 848ba8a19b86e6a65ace628f59502af6: Running coprocessor pre-open hook at 1733594518493Writing region info on filesystem at 1733594518493Initializing all the Stores at 1733594518494 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594518494Cleaning up temporary data from old regions at 1733594518508 (+14 ms)Running coprocessor post-open hooks at 1733594518517 (+9 ms)Region opened successfully at 1733594518518 (+1 ms) 2024-12-07T18:01:58,519 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6., pid=6, masterSystemTime=1733594518485 2024-12-07T18:01:58,523 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:58,523 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:58,523 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=848ba8a19b86e6a65ace628f59502af6, regionState=OPEN, openSeqNum=2, regionLocation=bd53b59592b3,39593,1733594515616 2024-12-07T18:01:58,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 848ba8a19b86e6a65ace628f59502af6, server=bd53b59592b3,39593,1733594515616 because future has completed 2024-12-07T18:01:58,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T18:01:58,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 848ba8a19b86e6a65ace628f59502af6, server=bd53b59592b3,39593,1733594515616 in 197 msec 2024-12-07T18:01:58,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T18:01:58,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, ASSIGN in 363 msec 2024-12-07T18:01:58,537 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733594518078 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T18:01:58,537 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733594518078","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733594518537"}]},"ts":"1733594518537"} 2024-12-07T18:01:58,541 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733594518078, state=ENABLED in hbase:meta 2024-12-07T18:01:58,542 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733594518078 execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T18:01:58,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733594518078 in 434 msec 2024-12-07T18:01:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:01:58,753 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733594518078 get assigned. Timeout = 60000ms 2024-12-07T18:01:58,753 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733594518078 completed 2024-12-07T18:01:58,758 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T18:01:58,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733594518078 assigned to meta. Checking AM states. 2024-12-07T18:01:58,765 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T18:01:58,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733594518078 assigned. 2024-12-07T18:01:58,776 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733594518078', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6., hostname=bd53b59592b3,39593,1733594515616, seqNum=2] 2024-12-07T18:01:58,778 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:01:58,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:01:58,791 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 848ba8a19b86e6a65ace628f59502af6 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-07T18:01:58,818 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cc740f839b0c4d369bc9bfda10af4eb9 2024-12-07T18:01:58,859 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/.tmp/family/cc740f839b0c4d369bc9bfda10af4eb9 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-07T18:01:58,866 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/.tmp/family/cc740f839b0c4d369bc9bfda10af4eb9 2024-12-07T18:01:58,910 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/.tmp/family/cc740f839b0c4d369bc9bfda10af4eb9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9 2024-12-07T18:01:58,912 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9, delay=939 ms 2024-12-07T18:01:58,915 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9, entries=2, sequenceid=6, filesize=4.9 K 2024-12-07T18:01:58,924 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 848ba8a19b86e6a65ace628f59502af6 in 130ms, sequenceid=6, compaction requested=false 2024-12-07T18:01:58,925 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733594518078' 2024-12-07T18:01:58,928 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 848ba8a19b86e6a65ace628f59502af6: 2024-12-07T18:01:59,854 DEBUG [hfile-prefetch-1733594518912 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key cc740f839b0c4d369bc9bfda10af4eb9_0. 2024-12-07T18:01:59,857 DEBUG [hfile-prefetch-1733594518912 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key cc740f839b0c4d369bc9bfda10af4eb9_117. 2024-12-07T18:01:59,858 DEBUG [hfile-prefetch-1733594518912 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9, with total blocks 2, and data blocks 1 2024-12-07T18:01:59,858 DEBUG [hfile-prefetch-1733594518912 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-07T18:01:59,858 DEBUG [hfile-prefetch-1733594518912 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-07T18:01:59,859 DEBUG [hfile-prefetch-1733594518912 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9. Will sleep 100ms and try the verification again. 2024-12-07T18:01:59,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=848ba8a19b86e6a65ace628f59502af6, source=bd53b59592b3,39593,1733594515616, destination=bd53b59592b3,42075,1733594515444, warming up region on bd53b59592b3,42075,1733594515444 2024-12-07T18:01:59,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=848ba8a19b86e6a65ace628f59502af6, source=bd53b59592b3,39593,1733594515616, destination=bd53b59592b3,42075,1733594515444, running balancer 2024-12-07T18:01:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, REOPEN/MOVE 2024-12-07T18:01:59,947 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, REOPEN/MOVE 2024-12-07T18:01:59,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(7855): Warmup {ENCODED => 848ba8a19b86e6a65ace628f59502af6, NAME => 'testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:01:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:01:59,949 INFO [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:01:59,949 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=848ba8a19b86e6a65ace628f59502af6, regionState=CLOSING, regionLocation=bd53b59592b3,39593,1733594515616 2024-12-07T18:01:59,949 INFO [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 848ba8a19b86e6a65ace628f59502af6 columnFamilyName family 2024-12-07T18:01:59,949 DEBUG [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:01:59,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, REOPEN/MOVE because future has completed 2024-12-07T18:01:59,953 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T18:01:59,953 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T18:01:59,953 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-07T18:01:59,953 DEBUG [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9 2024-12-07T18:01:59,953 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T18:01:59,953 INFO [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] regionserver.HStore(327): Store=848ba8a19b86e6a65ace628f59502af6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:01:59,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 848ba8a19b86e6a65ace628f59502af6, server=bd53b59592b3,39593,1733594515616}] 2024-12-07T18:01:59,953 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T18:01:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1722): Closing 848ba8a19b86e6a65ace628f59502af6, disabling compactions & flushes 2024-12-07T18:01:59,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. after waiting 0 ms 2024-12-07T18:01:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:59,954 DEBUG [StoreFileCloser-848ba8a19b86e6a65ace628f59502af6-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9 2024-12-07T18:01:59,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:01:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42075 {}] regionserver.HRegion(1676): Region close journal for 848ba8a19b86e6a65ace628f59502af6: Waiting for close lock at 1733594519953Disabling compacts and flushes for region at 1733594519953Disabling writes for close at 1733594519954 (+1 ms)Writing region close event to WAL at 1733594519955 (+1 ms)Closed at 1733594519955 2024-12-07T18:01:59,956 WARN [hfile-prefetch-1733594518912 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-07T18:01:59,957 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T18:01:59,957 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T18:01:59,957 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T18:01:59,957 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T18:01:59,957 DEBUG [hfile-prefetch-1733594518912 {}] hfile.PrefetchExecutor(121): Prefetch completed for cc740f839b0c4d369bc9bfda10af4eb9 2024-12-07T18:01:59,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T18:01:59,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733594519953 2024-12-07T18:01:59,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T18:01:59,971 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-07T18:01:59,971 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733594519957 2024-12-07T18:01:59,971 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T18:01:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34575 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-07T18:02:00,112 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,113 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-07T18:02:00,114 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 848ba8a19b86e6a65ace628f59502af6, disabling compactions & flushes 2024-12-07T18:02:00,114 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,114 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,114 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. after waiting 0 ms 2024-12-07T18:02:00,114 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,117 DEBUG [StoreFileCloser-848ba8a19b86e6a65ace628f59502af6-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file cc740f839b0c4d369bc9bfda10af4eb9, starting offset: 0, end offset: 9223372036854775807 2024-12-07T18:02:00,119 DEBUG [StoreFileCloser-848ba8a19b86e6a65ace628f59502af6-family-1 {}] bucket.BucketCache(856): removed key cc740f839b0c4d369bc9bfda10af4eb9_0 from back map with offset lock 2094080 in the evict process 2024-12-07T18:02:00,125 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-07T18:02:00,126 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,126 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 848ba8a19b86e6a65ace628f59502af6: Waiting for close lock at 1733594520113Running coprocessor pre-close hooks at 1733594520113Disabling compacts and flushes for region at 1733594520114 (+1 ms)Disabling writes for close at 1733594520114Writing region close event to WAL at 1733594520120 (+6 ms)Running coprocessor post-close hooks at 1733594520125 (+5 ms)Closed at 1733594520126 (+1 ms) 2024-12-07T18:02:00,126 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 848ba8a19b86e6a65ace628f59502af6 move to bd53b59592b3,42075,1733594515444 record at close sequenceid=6 2024-12-07T18:02:00,129 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,130 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=848ba8a19b86e6a65ace628f59502af6, regionState=CLOSED 2024-12-07T18:02:00,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 848ba8a19b86e6a65ace628f59502af6, server=bd53b59592b3,39593,1733594515616 because future has completed 2024-12-07T18:02:00,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T18:02:00,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 848ba8a19b86e6a65ace628f59502af6, server=bd53b59592b3,39593,1733594515616 in 180 msec 2024-12-07T18:02:00,136 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, REOPEN/MOVE; state=CLOSED, location=bd53b59592b3,42075,1733594515444; forceNewPlan=false, retain=false 2024-12-07T18:02:00,169 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T18:02:00,169 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T18:02:00,169 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T18:02:00,169 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T18:02:00,174 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T18:02:00,174 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733594520169 2024-12-07T18:02:00,174 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T18:02:00,287 INFO [bd53b59592b3:34575 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T18:02:00,287 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=848ba8a19b86e6a65ace628f59502af6, regionState=OPENING, regionLocation=bd53b59592b3,42075,1733594515444 2024-12-07T18:02:00,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, REOPEN/MOVE because future has completed 2024-12-07T18:02:00,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 848ba8a19b86e6a65ace628f59502af6, server=bd53b59592b3,42075,1733594515444}] 2024-12-07T18:02:00,449 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,450 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 848ba8a19b86e6a65ace628f59502af6, NAME => 'testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:02:00,451 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733594518078 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,451 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:02:00,451 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,451 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,452 INFO [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,453 INFO [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 848ba8a19b86e6a65ace628f59502af6 columnFamilyName family 2024-12-07T18:02:00,453 DEBUG [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:00,456 DEBUG [StoreFileOpener-848ba8a19b86e6a65ace628f59502af6-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9, delay=977 ms 2024-12-07T18:02:00,458 DEBUG [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9 2024-12-07T18:02:00,458 INFO [StoreOpener-848ba8a19b86e6a65ace628f59502af6-1 {}] regionserver.HStore(327): Store=848ba8a19b86e6a65ace628f59502af6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:02:00,458 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,460 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,460 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,460 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,460 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,467 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,468 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 848ba8a19b86e6a65ace628f59502af6; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60376051, jitterRate=-0.10032673180103302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T18:02:00,468 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,469 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 848ba8a19b86e6a65ace628f59502af6: Running coprocessor pre-open hook at 1733594520451Writing region info on filesystem at 1733594520451Initializing all the Stores at 1733594520452 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594520452Cleaning up temporary data from old regions at 1733594520461 (+9 ms)Running coprocessor post-open hooks at 1733594520468 (+7 ms)Region opened successfully at 1733594520469 (+1 ms) 2024-12-07T18:02:00,470 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6., pid=9, masterSystemTime=1733594520443 2024-12-07T18:02:00,473 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,473 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=848ba8a19b86e6a65ace628f59502af6, regionState=OPEN, openSeqNum=10, regionLocation=bd53b59592b3,42075,1733594515444 2024-12-07T18:02:00,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 848ba8a19b86e6a65ace628f59502af6, server=bd53b59592b3,42075,1733594515444 because future has completed 2024-12-07T18:02:00,478 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34575 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=bd53b59592b3,42075,1733594515444, table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T18:02:00,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-07T18:02:00,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 848ba8a19b86e6a65ace628f59502af6, server=bd53b59592b3,42075,1733594515444 in 188 msec 2024-12-07T18:02:00,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733594518078, region=848ba8a19b86e6a65ace628f59502af6, REOPEN/MOVE in 534 msec 2024-12-07T18:02:00,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T18:02:00,500 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:02:00,500 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:02:00,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:00,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:00,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:02:00,506 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T18:02:00,506 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2143720667, stopped=false 2024-12-07T18:02:00,506 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bd53b59592b3,34575,1733594514554 2024-12-07T18:02:00,532 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T18:02:00,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T18:02:00,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T18:02:00,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:00,532 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:00,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:00,533 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T18:02:00,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:02:00,534 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:02:00,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:02:00,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:02:00,534 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:02:00,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:00,536 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bd53b59592b3,42075,1733594515444' ***** 2024-12-07T18:02:00,536 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T18:02:00,536 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bd53b59592b3,39593,1733594515616' ***** 2024-12-07T18:02:00,537 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T18:02:00,537 INFO [RS:0;bd53b59592b3:42075 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T18:02:00,537 INFO [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T18:02:00,537 INFO [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.persistence 2024-12-07T18:02:00,537 INFO [RS:0;bd53b59592b3:42075 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/bucket.persistence 2024-12-07T18:02:00,537 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T18:02:00,537 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T18:02:00,537 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T18:02:00,537 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T18:02:00,538 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T18:02:00,538 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T18:02:00,538 INFO [RS:0;bd53b59592b3:42075 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T18:02:00,538 INFO [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T18:02:00,538 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T18:02:00,538 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T18:02:00,548 DEBUG [RS:0;bd53b59592b3:42075 {}] bucket.BucketCache(1487): Thread RS:0;bd53b59592b3:42075 started persisting bucket cache to file 2024-12-07T18:02:00,548 DEBUG [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1487): Thread RS:1;bd53b59592b3:39593 started persisting bucket cache to file 2024-12-07T18:02:00,548 DEBUG [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T18:02:00,548 DEBUG [RS:0;bd53b59592b3:42075 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T18:02:00,548 DEBUG [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T18:02:00,548 DEBUG [RS:0;bd53b59592b3:42075 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T18:02:00,551 DEBUG [RS:0;bd53b59592b3:42075 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T18:02:00,551 DEBUG [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T18:02:00,551 DEBUG [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733594520548 2024-12-07T18:02:00,551 DEBUG [RS:0;bd53b59592b3:42075 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733594520548 2024-12-07T18:02:00,552 DEBUG [RS:0;bd53b59592b3:42075 {}] bucket.BucketCache(1509): Thread RS:0;bd53b59592b3:42075 finished persisting bucket cache to file, renaming 2024-12-07T18:02:00,552 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T18:02:00,552 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T18:02:00,552 INFO [RS:0;bd53b59592b3:42075 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T18:02:00,552 INFO [RS:0;bd53b59592b3:42075 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T18:02:00,552 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(3091): Received CLOSE for 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,553 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(959): stopping server bd53b59592b3,42075,1733594515444 2024-12-07T18:02:00,553 INFO [RS:0;bd53b59592b3:42075 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T18:02:00,553 INFO [RS:0;bd53b59592b3:42075 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bd53b59592b3:42075. 2024-12-07T18:02:00,553 DEBUG [RS:0;bd53b59592b3:42075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:02:00,553 DEBUG [RS:0;bd53b59592b3:42075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:00,553 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T18:02:00,553 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T18:02:00,554 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 848ba8a19b86e6a65ace628f59502af6, disabling compactions & flushes 2024-12-07T18:02:00,554 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T18:02:00,554 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,554 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T18:02:00,554 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,554 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T18:02:00,554 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. after waiting 0 ms 2024-12-07T18:02:00,554 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,554 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-07T18:02:00,554 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T18:02:00,554 DEBUG [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 848ba8a19b86e6a65ace628f59502af6=testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6.} 2024-12-07T18:02:00,554 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T18:02:00,554 DEBUG [StoreFileCloser-848ba8a19b86e6a65ace628f59502af6-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/family/cc740f839b0c4d369bc9bfda10af4eb9 2024-12-07T18:02:00,554 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T18:02:00,555 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T18:02:00,555 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T18:02:00,555 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T18:02:00,555 DEBUG [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 848ba8a19b86e6a65ace628f59502af6 2024-12-07T18:02:00,555 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-07T18:02:00,559 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnRegionMove1733594518078/848ba8a19b86e6a65ace628f59502af6/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-07T18:02:00,559 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,559 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 848ba8a19b86e6a65ace628f59502af6: Waiting for close lock at 1733594520553Running coprocessor pre-close hooks at 1733594520553Disabling compacts and flushes for region at 1733594520553Disabling writes for close at 1733594520554 (+1 ms)Writing region close event to WAL at 1733594520555 (+1 ms)Running coprocessor post-close hooks at 1733594520559 (+4 ms)Closed at 1733594520559 2024-12-07T18:02:00,560 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6. 2024-12-07T18:02:00,565 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bc5a433615494b7d98a78624542f4877 2024-12-07T18:02:00,566 DEBUG [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1509): Thread RS:1;bd53b59592b3:39593 finished persisting bucket cache to file, renaming 2024-12-07T18:02:00,566 WARN [RS:1;bd53b59592b3:39593 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-07T18:02:00,566 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T18:02:00,566 INFO [RS:1;bd53b59592b3:39593 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T18:02:00,566 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T18:02:00,566 INFO [RS:1;bd53b59592b3:39593 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T18:02:00,566 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(959): stopping server bd53b59592b3,39593,1733594515616 2024-12-07T18:02:00,566 INFO [RS:1;bd53b59592b3:39593 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T18:02:00,566 INFO [RS:1;bd53b59592b3:39593 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;bd53b59592b3:39593. 2024-12-07T18:02:00,566 DEBUG [RS:1;bd53b59592b3:39593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:02:00,566 DEBUG [RS:1;bd53b59592b3:39593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:00,567 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(976): stopping server bd53b59592b3,39593,1733594515616; all regions closed. 2024-12-07T18:02:00,571 DEBUG [RS:1;bd53b59592b3:39593 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs 2024-12-07T18:02:00,571 INFO [RS:1;bd53b59592b3:39593 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bd53b59592b3%2C39593%2C1733594515616:(num 1733594517288) 2024-12-07T18:02:00,572 DEBUG [RS:1;bd53b59592b3:39593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:00,572 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:00,572 INFO [RS:1;bd53b59592b3:39593 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T18:02:00,572 INFO [RS:1;bd53b59592b3:39593 {}] hbase.ChoreService(370): Chore service for: regionserver/bd53b59592b3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T18:02:00,572 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T18:02:00,572 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T18:02:00,572 INFO [regionserver/bd53b59592b3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T18:02:00,572 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T18:02:00,573 INFO [RS:1;bd53b59592b3:39593 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T18:02:00,573 INFO [RS:1;bd53b59592b3:39593 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39593 2024-12-07T18:02:00,582 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/info/bc5a433615494b7d98a78624542f4877 is 205, key is testBlockEvictionOnRegionMove1733594518078,,1733594518100.848ba8a19b86e6a65ace628f59502af6./info:regioninfo/1733594520473/Put/seqid=0 2024-12-07T18:02:00,584 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/info/bc5a433615494b7d98a78624542f4877 2024-12-07T18:02:00,592 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 882b18badd674ee185c0692e614fd06e 2024-12-07T18:02:00,598 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bd53b59592b3,39593,1733594515616 2024-12-07T18:02:00,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:02:00,598 INFO [RS:1;bd53b59592b3:39593 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T18:02:00,605 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/ns/882b18badd674ee185c0692e614fd06e is 43, key is default/ns:d/1733594517874/Put/seqid=0 2024-12-07T18:02:00,606 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/ns/882b18badd674ee185c0692e614fd06e 2024-12-07T18:02:00,607 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bd53b59592b3,39593,1733594515616] 2024-12-07T18:02:00,614 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8cf25f894ec5452785273fe861f2d0ca 2024-12-07T18:02:00,615 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bd53b59592b3,39593,1733594515616 already deleted, retry=false 2024-12-07T18:02:00,615 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bd53b59592b3,39593,1733594515616 expired; onlineServers=1 2024-12-07T18:02:00,628 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/table/8cf25f894ec5452785273fe861f2d0ca is 78, key is testBlockEvictionOnRegionMove1733594518078/table:state/1733594518537/Put/seqid=0 2024-12-07T18:02:00,629 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/table/8cf25f894ec5452785273fe861f2d0ca 2024-12-07T18:02:00,630 INFO [regionserver/bd53b59592b3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:00,631 INFO [regionserver/bd53b59592b3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:00,632 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/info/bc5a433615494b7d98a78624542f4877 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/info/bc5a433615494b7d98a78624542f4877 2024-12-07T18:02:00,633 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/info/bc5a433615494b7d98a78624542f4877, delay=943 ms 2024-12-07T18:02:00,634 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/info/bc5a433615494b7d98a78624542f4877, entries=15, sequenceid=15, filesize=7.6 K 2024-12-07T18:02:00,635 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/ns/882b18badd674ee185c0692e614fd06e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/ns/882b18badd674ee185c0692e614fd06e 2024-12-07T18:02:00,636 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/ns/882b18badd674ee185c0692e614fd06e, delay=940 ms 2024-12-07T18:02:00,637 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/ns/882b18badd674ee185c0692e614fd06e, entries=2, sequenceid=15, filesize=5.0 K 2024-12-07T18:02:00,637 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/table/8cf25f894ec5452785273fe861f2d0ca as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/table/8cf25f894ec5452785273fe861f2d0ca 2024-12-07T18:02:00,638 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/table/8cf25f894ec5452785273fe861f2d0ca, delay=903 ms 2024-12-07T18:02:00,639 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/table/8cf25f894ec5452785273fe861f2d0ca, entries=2, sequenceid=15, filesize=5.3 K 2024-12-07T18:02:00,640 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=15, compaction requested=false 2024-12-07T18:02:00,640 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-07T18:02:00,642 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/info/bc5a433615494b7d98a78624542f4877 2024-12-07T18:02:00,642 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/ns/882b18badd674ee185c0692e614fd06e 2024-12-07T18:02:00,643 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/table/8cf25f894ec5452785273fe861f2d0ca 2024-12-07T18:02:00,647 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-07T18:02:00,648 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T18:02:00,648 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T18:02:00,648 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733594520554Running coprocessor pre-close hooks at 1733594520554Disabling compacts and flushes for region at 1733594520554Disabling writes for close at 1733594520555 (+1 ms)Obtaining lock to block concurrent updates at 1733594520555Preparing flush snapshotting stores in 1588230740 at 1733594520555Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733594520556 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733594520556Flushing 1588230740/info: creating writer at 1733594520556Flushing 1588230740/info: appending metadata at 1733594520581 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733594520581Flushing 1588230740/ns: creating writer at 1733594520587 (+6 ms)Flushing 1588230740/ns: appending metadata at 1733594520604 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733594520604Flushing 1588230740/table: creating writer at 1733594520609 (+5 ms)Flushing 1588230740/table: appending metadata at 1733594520628 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733594520628Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ef68e04: reopening flushed file at 1733594520632 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b14be35: reopening flushed file at 1733594520634 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78e22b86: reopening flushed file at 1733594520637 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=15, compaction requested=false at 1733594520640 (+3 ms)Writing region close event to WAL at 1733594520643 (+3 ms)Running coprocessor post-close hooks at 1733594520648 (+5 ms)Closed at 1733594520648 2024-12-07T18:02:00,649 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T18:02:00,707 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:00,707 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x100017ec1ea0002, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:00,708 INFO [RS:1;bd53b59592b3:39593 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T18:02:00,708 INFO [RS:1;bd53b59592b3:39593 {}] regionserver.HRegionServer(1031): Exiting; stopping=bd53b59592b3,39593,1733594515616; zookeeper connection closed. 2024-12-07T18:02:00,708 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@689830c2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@689830c2 2024-12-07T18:02:00,755 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(976): stopping server bd53b59592b3,42075,1733594515444; all regions closed. 2024-12-07T18:02:00,761 DEBUG [RS:0;bd53b59592b3:42075 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs 2024-12-07T18:02:00,761 INFO [RS:0;bd53b59592b3:42075 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bd53b59592b3%2C42075%2C1733594515444.meta:.meta(num 1733594517701) 2024-12-07T18:02:00,764 DEBUG [RS:0;bd53b59592b3:42075 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs 2024-12-07T18:02:00,764 INFO [RS:0;bd53b59592b3:42075 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bd53b59592b3%2C42075%2C1733594515444:(num 1733594517288) 2024-12-07T18:02:00,764 DEBUG [RS:0;bd53b59592b3:42075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:00,764 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:00,765 INFO [RS:0;bd53b59592b3:42075 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T18:02:00,765 INFO [RS:0;bd53b59592b3:42075 {}] hbase.ChoreService(370): Chore service for: regionserver/bd53b59592b3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T18:02:00,765 INFO [RS:0;bd53b59592b3:42075 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T18:02:00,765 INFO [regionserver/bd53b59592b3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T18:02:00,766 INFO [RS:0;bd53b59592b3:42075 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42075 2024-12-07T18:02:00,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bd53b59592b3,42075,1733594515444 2024-12-07T18:02:00,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:02:00,788 INFO [RS:0;bd53b59592b3:42075 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T18:02:00,789 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bd53b59592b3,42075,1733594515444] 2024-12-07T18:02:00,806 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bd53b59592b3,42075,1733594515444 already deleted, retry=false 2024-12-07T18:02:00,806 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bd53b59592b3,42075,1733594515444 expired; onlineServers=0 2024-12-07T18:02:00,806 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bd53b59592b3,34575,1733594514554' ***** 2024-12-07T18:02:00,806 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T18:02:00,807 INFO [M:0;bd53b59592b3:34575 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T18:02:00,807 INFO [M:0;bd53b59592b3:34575 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T18:02:00,807 DEBUG [M:0;bd53b59592b3:34575 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T18:02:00,807 DEBUG [M:0;bd53b59592b3:34575 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T18:02:00,807 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T18:02:00,807 DEBUG [master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.small.0-1733594516884 {}] cleaner.HFileCleaner(306): Exit Thread[master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.small.0-1733594516884,5,FailOnTimeoutGroup] 2024-12-07T18:02:00,807 DEBUG [master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.large.0-1733594516883 {}] cleaner.HFileCleaner(306): Exit Thread[master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.large.0-1733594516883,5,FailOnTimeoutGroup] 2024-12-07T18:02:00,807 INFO [M:0;bd53b59592b3:34575 {}] hbase.ChoreService(370): Chore service for: master/bd53b59592b3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T18:02:00,808 INFO [M:0;bd53b59592b3:34575 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T18:02:00,808 DEBUG [M:0;bd53b59592b3:34575 {}] master.HMaster(1795): Stopping service threads 2024-12-07T18:02:00,808 INFO [M:0;bd53b59592b3:34575 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T18:02:00,808 INFO [M:0;bd53b59592b3:34575 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T18:02:00,808 INFO [M:0;bd53b59592b3:34575 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T18:02:00,809 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T18:02:00,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T18:02:00,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:00,815 DEBUG [M:0;bd53b59592b3:34575 {}] zookeeper.ZKUtil(347): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T18:02:00,815 WARN [M:0;bd53b59592b3:34575 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T18:02:00,815 INFO [M:0;bd53b59592b3:34575 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/.lastflushedseqids 2024-12-07T18:02:00,825 INFO [M:0;bd53b59592b3:34575 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T18:02:00,825 INFO [M:0;bd53b59592b3:34575 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T18:02:00,826 DEBUG [M:0;bd53b59592b3:34575 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T18:02:00,826 INFO [M:0;bd53b59592b3:34575 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:00,826 DEBUG [M:0;bd53b59592b3:34575 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:00,826 DEBUG [M:0;bd53b59592b3:34575 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T18:02:00,826 DEBUG [M:0;bd53b59592b3:34575 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:00,826 INFO [M:0;bd53b59592b3:34575 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-12-07T18:02:00,833 DEBUG [M:0;bd53b59592b3:34575 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 44bd6764387b480eae80edaa66e5af6f 2024-12-07T18:02:00,846 DEBUG [M:0;bd53b59592b3:34575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/44bd6764387b480eae80edaa66e5af6f is 82, key is hbase:meta,,1/info:regioninfo/1733594517775/Put/seqid=0 2024-12-07T18:02:00,847 INFO [M:0;bd53b59592b3:34575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/44bd6764387b480eae80edaa66e5af6f 2024-12-07T18:02:00,854 DEBUG [M:0;bd53b59592b3:34575 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7bb722329e6e4be4a53403e7222d5bae 2024-12-07T18:02:00,866 DEBUG [M:0;bd53b59592b3:34575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7bb722329e6e4be4a53403e7222d5bae is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733594518544/Put/seqid=0 2024-12-07T18:02:00,866 INFO [M:0;bd53b59592b3:34575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7bb722329e6e4be4a53403e7222d5bae 2024-12-07T18:02:00,873 DEBUG [M:0;bd53b59592b3:34575 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 03937d17eb204e688a39e63fbf541a48 2024-12-07T18:02:00,885 DEBUG [M:0;bd53b59592b3:34575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03937d17eb204e688a39e63fbf541a48 is 69, key is bd53b59592b3,39593,1733594515616/rs:state/1733594516962/Put/seqid=0 2024-12-07T18:02:00,885 INFO [M:0;bd53b59592b3:34575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03937d17eb204e688a39e63fbf541a48 2024-12-07T18:02:00,887 DEBUG [M:0;bd53b59592b3:34575 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/44bd6764387b480eae80edaa66e5af6f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/44bd6764387b480eae80edaa66e5af6f 2024-12-07T18:02:00,889 INFO [M:0;bd53b59592b3:34575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/44bd6764387b480eae80edaa66e5af6f, entries=8, sequenceid=84, filesize=5.5 K 2024-12-07T18:02:00,889 DEBUG [M:0;bd53b59592b3:34575 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7bb722329e6e4be4a53403e7222d5bae as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7bb722329e6e4be4a53403e7222d5bae 2024-12-07T18:02:00,890 INFO [M:0;bd53b59592b3:34575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7bb722329e6e4be4a53403e7222d5bae, entries=9, sequenceid=84, filesize=6.5 K 2024-12-07T18:02:00,891 DEBUG [M:0;bd53b59592b3:34575 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03937d17eb204e688a39e63fbf541a48 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03937d17eb204e688a39e63fbf541a48 2024-12-07T18:02:00,892 INFO [M:0;bd53b59592b3:34575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03937d17eb204e688a39e63fbf541a48, entries=2, sequenceid=84, filesize=5.1 K 2024-12-07T18:02:00,893 INFO [M:0;bd53b59592b3:34575 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=84, compaction requested=false 2024-12-07T18:02:00,895 INFO [M:0;bd53b59592b3:34575 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:00,895 DEBUG [M:0;bd53b59592b3:34575 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733594520826Disabling compacts and flushes for region at 1733594520826Disabling writes for close at 1733594520826Obtaining lock to block concurrent updates at 1733594520826Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733594520826Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733594520827 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733594520827Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733594520827Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733594520845 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733594520846 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733594520849 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733594520865 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733594520865Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733594520868 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733594520884 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733594520885 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37b87542: reopening flushed file at 1733594520887 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d45f28b: reopening flushed file at 1733594520889 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33c6155: reopening flushed file at 1733594520891 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=84, compaction requested=false at 1733594520893 (+2 ms)Writing region close event to WAL at 1733594520895 (+2 ms)Closed at 1733594520895 2024-12-07T18:02:00,896 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T18:02:00,896 INFO [M:0;bd53b59592b3:34575 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T18:02:00,897 INFO [M:0;bd53b59592b3:34575 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34575 2024-12-07T18:02:00,897 INFO [M:0;bd53b59592b3:34575 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T18:02:00,898 INFO [RS:0;bd53b59592b3:42075 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T18:02:00,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:00,898 INFO [RS:0;bd53b59592b3:42075 {}] regionserver.HRegionServer(1031): Exiting; stopping=bd53b59592b3,42075,1733594515444; zookeeper connection closed. 2024-12-07T18:02:00,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42075-0x100017ec1ea0001, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:00,898 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67646274 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67646274 2024-12-07T18:02:00,899 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-07T18:02:01,032 INFO [M:0;bd53b59592b3:34575 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T18:02:01,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:01,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34575-0x100017ec1ea0000, quorum=127.0.0.1:56707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:01,037 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T18:02:01,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-07T18:02:01,060 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:01,067 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733594518912 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733594520633 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/bd53b59592b3:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/bd53b59592b3:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/bd53b59592b3:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/bd53b59592b3:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733594520636 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733594520456 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=354 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=20452 (was 20956) 2024-12-07T18:02:01,072 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=354, ProcessCount=11, AvailableMemoryMB=20452 2024-12-07T18:02:01,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/hadoop-log-dir so I do NOT create it in target/test-data/14319508-4508-aeac-b191-3e8f10296d47 2024-12-07T18:02:01,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/hadoop-tmp-dir so I do NOT create it in target/test-data/14319508-4508-aeac-b191-3e8f10296d47 2024-12-07T18:02:01,073 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:01,075 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/cluster_de315eda-609c-fb6d-28e1-8039a7259b3a, deleteOnExit=true 2024-12-07T18:02:01,109 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/cluster_de315eda-609c-fb6d-28e1-8039a7259b3a/zookeeper_0, clientPort=62026, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/cluster_de315eda-609c-fb6d-28e1-8039a7259b3a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/cluster_de315eda-609c-fb6d-28e1-8039a7259b3a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T18:02:01,111 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62026 2024-12-07T18:02:01,127 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 with version=8 2024-12-07T18:02:01,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/hbase-staging 2024-12-07T18:02:01,130 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bd53b59592b3:0 server-side Connection retries=45 2024-12-07T18:02:01,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:01,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:01,130 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T18:02:01,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:01,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T18:02:01,131 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T18:02:01,131 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T18:02:01,132 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40469 2024-12-07T18:02:01,133 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40469 connecting to ZooKeeper ensemble=127.0.0.1:62026 2024-12-07T18:02:01,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:404690x0, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T18:02:01,172 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40469-0x100017edcb00000 connected 2024-12-07T18:02:01,250 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:01,251 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:01,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:02:01,253 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9, hbase.cluster.distributed=false 2024-12-07T18:02:01,257 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T18:02:01,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40469 2024-12-07T18:02:01,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40469 2024-12-07T18:02:01,258 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40469 2024-12-07T18:02:01,258 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40469 2024-12-07T18:02:01,259 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40469 2024-12-07T18:02:01,275 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bd53b59592b3:0 server-side Connection retries=45 2024-12-07T18:02:01,276 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:01,276 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:01,276 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T18:02:01,276 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:01,276 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T18:02:01,276 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T18:02:01,276 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T18:02:01,277 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42669 2024-12-07T18:02:01,278 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42669 connecting to ZooKeeper ensemble=127.0.0.1:62026 2024-12-07T18:02:01,278 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:01,278 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:01,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:426690x0, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T18:02:01,292 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:426690x0, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:02:01,292 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42669-0x100017edcb00001 connected 2024-12-07T18:02:01,292 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T18:02:01,295 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.cache 2024-12-07T18:02:01,295 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T18:02:01,295 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T18:02:01,295 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T18:02:01,296 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T18:02:01,296 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T18:02:01,296 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T18:02:01,296 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T18:02:01,296 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.persistence is complete. 2024-12-07T18:02:01,296 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T18:02:01,297 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T18:02:01,299 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T18:02:01,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42669 2024-12-07T18:02:01,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42669 2024-12-07T18:02:01,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42669 2024-12-07T18:02:01,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42669 2024-12-07T18:02:01,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42669 2024-12-07T18:02:01,315 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bd53b59592b3:0 server-side Connection retries=45 2024-12-07T18:02:01,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:01,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:01,315 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T18:02:01,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:01,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T18:02:01,315 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T18:02:01,316 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T18:02:01,316 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33515 2024-12-07T18:02:01,318 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33515 connecting to ZooKeeper ensemble=127.0.0.1:62026 2024-12-07T18:02:01,318 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:01,319 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:01,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335150x0, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T18:02:01,332 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33515-0x100017edcb00002 connected 2024-12-07T18:02:01,332 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:02:01,332 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T18:02:01,333 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.cache 2024-12-07T18:02:01,333 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T18:02:01,333 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T18:02:01,333 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T18:02:01,334 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T18:02:01,334 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T18:02:01,334 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T18:02:01,334 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T18:02:01,334 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.persistence is complete. 2024-12-07T18:02:01,339 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T18:02:01,339 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T18:02:01,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T18:02:01,343 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33515 2024-12-07T18:02:01,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33515 2024-12-07T18:02:01,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33515 2024-12-07T18:02:01,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33515 2024-12-07T18:02:01,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33515 2024-12-07T18:02:01,357 DEBUG [M:0;bd53b59592b3:40469 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bd53b59592b3:40469 2024-12-07T18:02:01,357 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bd53b59592b3,40469,1733594521129 2024-12-07T18:02:01,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:02:01,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:02:01,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:02:01,367 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bd53b59592b3,40469,1733594521129 2024-12-07T18:02:01,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T18:02:01,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T18:02:01,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,375 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T18:02:01,375 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bd53b59592b3,40469,1733594521129 from backup master directory 2024-12-07T18:02:01,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bd53b59592b3,40469,1733594521129 2024-12-07T18:02:01,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:02:01,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:02:01,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:02:01,383 WARN [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T18:02:01,383 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bd53b59592b3,40469,1733594521129 2024-12-07T18:02:01,385 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/hbase.id] with ID: 6c83292e-7a73-4c09-872c-83932733bea9 2024-12-07T18:02:01,385 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/.tmp/hbase.id 2024-12-07T18:02:01,390 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/hbase.id] 2024-12-07T18:02:01,424 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:01,424 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T18:02:01,427 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-12-07T18:02:01,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,445 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:02:01,447 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T18:02:01,447 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:02:01,470 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store 2024-12-07T18:02:01,476 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:02:01,476 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T18:02:01,476 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:01,476 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:01,476 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T18:02:01,476 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:01,476 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:01,477 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733594521476Disabling compacts and flushes for region at 1733594521476Disabling writes for close at 1733594521476Writing region close event to WAL at 1733594521476Closed at 1733594521476 2024-12-07T18:02:01,477 WARN [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/.initializing 2024-12-07T18:02:01,477 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/WALs/bd53b59592b3,40469,1733594521129 2024-12-07T18:02:01,482 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T18:02:01,485 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd53b59592b3%2C40469%2C1733594521129, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/WALs/bd53b59592b3,40469,1733594521129, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/oldWALs, maxLogs=10 2024-12-07T18:02:01,503 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/WALs/bd53b59592b3,40469,1733594521129/bd53b59592b3%2C40469%2C1733594521129.1733594521485 2024-12-07T18:02:01,504 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T18:02:01,504 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:02:01,504 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:02:01,504 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,504 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,508 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T18:02:01,509 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:01,509 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:02:01,509 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,513 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T18:02:01,513 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:01,513 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:02:01,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T18:02:01,516 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:01,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:02:01,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,519 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T18:02:01,520 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:01,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:02:01,520 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,520 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,520 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,521 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,521 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,521 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T18:02:01,521 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:02:01,527 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:02:01,527 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68670907, jitterRate=0.023276254534721375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T18:02:01,528 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733594521504Initializing all the Stores at 1733594521504Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594521504Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594521505 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594521505Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594521505Cleaning up temporary data from old regions at 1733594521521 (+16 ms)Region opened successfully at 1733594521527 (+6 ms) 2024-12-07T18:02:01,528 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T18:02:01,532 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c73c915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd53b59592b3/172.17.0.2:0 2024-12-07T18:02:01,533 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T18:02:01,533 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T18:02:01,533 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T18:02:01,533 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T18:02:01,534 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T18:02:01,534 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-07T18:02:01,534 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T18:02:01,536 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T18:02:01,537 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T18:02:01,556 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T18:02:01,556 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T18:02:01,557 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T18:02:01,566 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T18:02:01,566 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T18:02:01,567 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T18:02:01,574 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T18:02:01,575 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T18:02:01,582 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T18:02:01,585 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T18:02:01,591 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T18:02:01,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T18:02:01,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T18:02:01,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T18:02:01,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,600 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bd53b59592b3,40469,1733594521129, sessionid=0x100017edcb00000, setting cluster-up flag (Was=false) 2024-12-07T18:02:01,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,641 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T18:02:01,642 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bd53b59592b3,40469,1733594521129 2024-12-07T18:02:01,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:01,682 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T18:02:01,683 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bd53b59592b3,40469,1733594521129 2024-12-07T18:02:01,684 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T18:02:01,685 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T18:02:01,686 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T18:02:01,686 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T18:02:01,686 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bd53b59592b3,40469,1733594521129 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T18:02:01,688 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bd53b59592b3:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:02:01,688 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bd53b59592b3:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:02:01,688 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bd53b59592b3:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:02:01,688 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bd53b59592b3:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:02:01,688 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bd53b59592b3:0, corePoolSize=10, maxPoolSize=10 2024-12-07T18:02:01,688 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,688 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bd53b59592b3:0, corePoolSize=2, maxPoolSize=2 2024-12-07T18:02:01,688 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,689 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733594551689 2024-12-07T18:02:01,690 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T18:02:01,690 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T18:02:01,690 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T18:02:01,690 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T18:02:01,690 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T18:02:01,690 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T18:02:01,690 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T18:02:01,690 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T18:02:01,690 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,690 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:01,690 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T18:02:01,690 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T18:02:01,691 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T18:02:01,691 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T18:02:01,691 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T18:02:01,691 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T18:02:01,695 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.large.0-1733594521691,5,FailOnTimeoutGroup] 2024-12-07T18:02:01,696 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.small.0-1733594521695,5,FailOnTimeoutGroup] 2024-12-07T18:02:01,696 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,696 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T18:02:01,696 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,696 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,709 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T18:02:01,709 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 2024-12-07T18:02:01,714 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:02:01,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T18:02:01,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T18:02:01,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:01,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:02:01,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T18:02:01,723 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T18:02:01,723 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:01,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:02:01,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T18:02:01,726 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T18:02:01,726 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:01,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:02:01,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T18:02:01,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T18:02:01,729 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:01,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:02:01,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T18:02:01,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740 2024-12-07T18:02:01,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740 2024-12-07T18:02:01,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T18:02:01,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T18:02:01,730 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T18:02:01,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T18:02:01,735 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:02:01,735 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73153781, jitterRate=0.09007628262042999}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T18:02:01,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733594521714Initializing all the Stores at 1733594521715 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594521715Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594521715Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594521715Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594521715Cleaning up temporary data from old regions at 1733594521730 (+15 ms)Region opened successfully at 1733594521736 (+6 ms) 2024-12-07T18:02:01,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T18:02:01,736 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T18:02:01,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T18:02:01,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T18:02:01,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T18:02:01,737 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T18:02:01,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733594521736Disabling compacts and flushes for region at 1733594521736Disabling writes for close at 1733594521736Writing region close event to WAL at 1733594521737 (+1 ms)Closed at 1733594521737 2024-12-07T18:02:01,738 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T18:02:01,738 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T18:02:01,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T18:02:01,739 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T18:02:01,740 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T18:02:01,749 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(746): ClusterId : 6c83292e-7a73-4c09-872c-83932733bea9 2024-12-07T18:02:01,749 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(746): ClusterId : 6c83292e-7a73-4c09-872c-83932733bea9 2024-12-07T18:02:01,749 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T18:02:01,749 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T18:02:01,784 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T18:02:01,784 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T18:02:01,784 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T18:02:01,784 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T18:02:01,792 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T18:02:01,792 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T18:02:01,793 DEBUG [RS:0;bd53b59592b3:42669 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399e2901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd53b59592b3/172.17.0.2:0 2024-12-07T18:02:01,793 DEBUG [RS:1;bd53b59592b3:33515 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@350348e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd53b59592b3/172.17.0.2:0 2024-12-07T18:02:01,805 DEBUG [RS:0;bd53b59592b3:42669 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bd53b59592b3:42669 2024-12-07T18:02:01,805 DEBUG [RS:1;bd53b59592b3:33515 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;bd53b59592b3:33515 2024-12-07T18:02:01,805 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T18:02:01,805 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T18:02:01,805 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T18:02:01,806 DEBUG [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T18:02:01,806 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T18:02:01,806 DEBUG [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T18:02:01,806 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd53b59592b3,40469,1733594521129 with port=33515, startcode=1733594521314 2024-12-07T18:02:01,806 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd53b59592b3,40469,1733594521129 with port=42669, startcode=1733594521275 2024-12-07T18:02:01,807 DEBUG [RS:0;bd53b59592b3:42669 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T18:02:01,807 DEBUG [RS:1;bd53b59592b3:33515 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T18:02:01,809 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T18:02:01,809 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40937, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T18:02:01,810 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40469 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bd53b59592b3,33515,1733594521314 2024-12-07T18:02:01,810 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40469 {}] master.ServerManager(517): Registering regionserver=bd53b59592b3,33515,1733594521314 2024-12-07T18:02:01,812 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40469 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bd53b59592b3,42669,1733594521275 2024-12-07T18:02:01,812 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40469 {}] master.ServerManager(517): Registering regionserver=bd53b59592b3,42669,1733594521275 2024-12-07T18:02:01,812 DEBUG [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 2024-12-07T18:02:01,812 DEBUG [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T18:02:01,812 DEBUG [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T18:02:01,813 DEBUG [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 2024-12-07T18:02:01,813 DEBUG [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T18:02:01,813 DEBUG [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T18:02:01,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:02:01,855 DEBUG [RS:1;bd53b59592b3:33515 {}] zookeeper.ZKUtil(111): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bd53b59592b3,33515,1733594521314 2024-12-07T18:02:01,855 WARN [RS:1;bd53b59592b3:33515 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T18:02:01,856 INFO [RS:1;bd53b59592b3:33515 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:02:01,856 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bd53b59592b3,33515,1733594521314] 2024-12-07T18:02:01,856 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bd53b59592b3,42669,1733594521275] 2024-12-07T18:02:01,856 DEBUG [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,33515,1733594521314 2024-12-07T18:02:01,856 DEBUG [RS:0;bd53b59592b3:42669 {}] zookeeper.ZKUtil(111): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bd53b59592b3,42669,1733594521275 2024-12-07T18:02:01,856 WARN [RS:0;bd53b59592b3:42669 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T18:02:01,856 INFO [RS:0;bd53b59592b3:42669 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:02:01,857 DEBUG [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42669,1733594521275 2024-12-07T18:02:01,865 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T18:02:01,865 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T18:02:01,867 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T18:02:01,868 INFO [RS:1;bd53b59592b3:33515 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T18:02:01,868 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,868 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T18:02:01,869 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T18:02:01,869 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T18:02:01,869 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,870 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,870 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,870 INFO [RS:0;bd53b59592b3:42669 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T18:02:01,870 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,870 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,870 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,870 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,870 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T18:02:01,870 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bd53b59592b3:0, corePoolSize=2, maxPoolSize=2 2024-12-07T18:02:01,870 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,870 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T18:02:01,871 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,871 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:02:01,871 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,871 DEBUG [RS:1;bd53b59592b3:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:02:01,871 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bd53b59592b3:0, corePoolSize=2, maxPoolSize=2 2024-12-07T18:02:01,871 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,872 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,872 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,872 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,872 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,872 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:01,872 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:02:01,872 DEBUG [RS:0;bd53b59592b3:42669 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:02:01,873 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,33515,1733594521314-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,873 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,874 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,42669,1733594521275-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T18:02:01,887 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T18:02:01,887 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,33515,1733594521314-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,887 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,887 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.Replication(171): bd53b59592b3,33515,1733594521314 started 2024-12-07T18:02:01,890 WARN [bd53b59592b3:40469 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T18:02:01,899 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,899 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(1482): Serving as bd53b59592b3,33515,1733594521314, RpcServer on bd53b59592b3/172.17.0.2:33515, sessionid=0x100017edcb00002 2024-12-07T18:02:01,899 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T18:02:01,899 DEBUG [RS:1;bd53b59592b3:33515 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bd53b59592b3,33515,1733594521314 2024-12-07T18:02:01,899 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,33515,1733594521314' 2024-12-07T18:02:01,899 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T18:02:01,900 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T18:02:01,900 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T18:02:01,900 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,42669,1733594521275-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,901 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T18:02:01,901 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,901 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T18:02:01,901 DEBUG [RS:1;bd53b59592b3:33515 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bd53b59592b3,33515,1733594521314 2024-12-07T18:02:01,901 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.Replication(171): bd53b59592b3,42669,1733594521275 started 2024-12-07T18:02:01,901 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,33515,1733594521314' 2024-12-07T18:02:01,901 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T18:02:01,901 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T18:02:01,902 DEBUG [RS:1;bd53b59592b3:33515 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T18:02:01,902 INFO [RS:1;bd53b59592b3:33515 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T18:02:01,902 INFO [RS:1;bd53b59592b3:33515 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T18:02:01,916 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:01,916 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(1482): Serving as bd53b59592b3,42669,1733594521275, RpcServer on bd53b59592b3/172.17.0.2:42669, sessionid=0x100017edcb00001 2024-12-07T18:02:01,916 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T18:02:01,916 DEBUG [RS:0;bd53b59592b3:42669 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bd53b59592b3,42669,1733594521275 2024-12-07T18:02:01,916 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,42669,1733594521275' 2024-12-07T18:02:01,916 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T18:02:01,917 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T18:02:01,918 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T18:02:01,918 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T18:02:01,918 DEBUG [RS:0;bd53b59592b3:42669 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bd53b59592b3,42669,1733594521275 2024-12-07T18:02:01,918 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,42669,1733594521275' 2024-12-07T18:02:01,918 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T18:02:01,919 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T18:02:01,919 DEBUG [RS:0;bd53b59592b3:42669 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T18:02:01,919 INFO [RS:0;bd53b59592b3:42669 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T18:02:01,919 INFO [RS:0;bd53b59592b3:42669 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T18:02:02,003 INFO [RS:1;bd53b59592b3:33515 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T18:02:02,004 INFO [RS:1;bd53b59592b3:33515 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bd53b59592b3%2C33515%2C1733594521314, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,33515,1733594521314, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs, maxLogs=55 2024-12-07T18:02:02,020 INFO [RS:0;bd53b59592b3:42669 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T18:02:02,021 INFO [RS:0;bd53b59592b3:42669 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bd53b59592b3%2C42669%2C1733594521275, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42669,1733594521275, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs, maxLogs=55 2024-12-07T18:02:02,026 INFO [RS:1;bd53b59592b3:33515 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,33515,1733594521314/bd53b59592b3%2C33515%2C1733594521314.1733594522006 2024-12-07T18:02:02,026 DEBUG [RS:1;bd53b59592b3:33515 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T18:02:02,037 INFO [RS:0;bd53b59592b3:42669 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42669,1733594521275/bd53b59592b3%2C42669%2C1733594521275.1733594522021 2024-12-07T18:02:02,038 DEBUG [RS:0;bd53b59592b3:42669 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T18:02:02,140 DEBUG [bd53b59592b3:40469 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T18:02:02,141 DEBUG [bd53b59592b3:40469 {}] balancer.BalancerClusterState(204): Hosts are {bd53b59592b3=0} racks are {/default-rack=0} 2024-12-07T18:02:02,142 DEBUG [bd53b59592b3:40469 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T18:02:02,142 DEBUG [bd53b59592b3:40469 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T18:02:02,142 DEBUG [bd53b59592b3:40469 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T18:02:02,142 DEBUG [bd53b59592b3:40469 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T18:02:02,142 INFO [bd53b59592b3:40469 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T18:02:02,142 INFO [bd53b59592b3:40469 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T18:02:02,142 DEBUG [bd53b59592b3:40469 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T18:02:02,142 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bd53b59592b3,42669,1733594521275 2024-12-07T18:02:02,144 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd53b59592b3,42669,1733594521275, state=OPENING 2024-12-07T18:02:02,173 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T18:02:02,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:02,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:02,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:02,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:02:02,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:02:02,184 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T18:02:02,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bd53b59592b3,42669,1733594521275}] 2024-12-07T18:02:02,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:02:02,341 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T18:02:02,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36911, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T18:02:02,347 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T18:02:02,347 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:02:02,348 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T18:02:02,349 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bd53b59592b3%2C42669%2C1733594521275.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42669,1733594521275, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs, maxLogs=55 2024-12-07T18:02:02,366 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,42669,1733594521275/bd53b59592b3%2C42669%2C1733594521275.meta.1733594522350.meta 2024-12-07T18:02:02,367 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T18:02:02,367 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:02:02,367 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T18:02:02,367 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T18:02:02,367 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T18:02:02,367 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T18:02:02,368 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:02:02,368 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T18:02:02,368 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T18:02:02,368 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T18:02:02,369 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T18:02:02,369 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:02,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:02:02,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T18:02:02,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T18:02:02,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:02,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:02:02,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T18:02:02,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T18:02:02,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:02,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:02:02,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T18:02:02,371 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T18:02:02,371 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:02,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:02:02,371 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T18:02:02,372 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740 2024-12-07T18:02:02,372 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740 2024-12-07T18:02:02,372 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T18:02:02,373 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T18:02:02,373 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T18:02:02,373 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T18:02:02,374 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74937845, jitterRate=0.11666090786457062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T18:02:02,374 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T18:02:02,374 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733594522368Writing region info on filesystem at 1733594522368Initializing all the Stores at 1733594522368Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594522368Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594522368Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594522368Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733594522368Cleaning up temporary data from old regions at 1733594522373 (+5 ms)Running coprocessor post-open hooks at 1733594522374 (+1 ms)Region opened successfully at 1733594522374 2024-12-07T18:02:02,375 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733594522339 2024-12-07T18:02:02,378 DEBUG [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T18:02:02,378 INFO [RS_OPEN_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T18:02:02,378 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bd53b59592b3,42669,1733594521275 2024-12-07T18:02:02,379 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd53b59592b3,42669,1733594521275, state=OPEN 2024-12-07T18:02:02,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T18:02:02,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T18:02:02,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T18:02:02,406 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bd53b59592b3,42669,1733594521275 2024-12-07T18:02:02,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:02:02,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:02:02,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:02:02,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T18:02:02,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd53b59592b3,42669,1733594521275 in 222 msec 2024-12-07T18:02:02,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T18:02:02,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 671 msec 2024-12-07T18:02:02,411 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T18:02:02,411 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T18:02:02,414 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:02:02,414 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd53b59592b3,42669,1733594521275, seqNum=-1] 2024-12-07T18:02:02,414 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:02:02,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:02:02,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 736 msec 2024-12-07T18:02:02,422 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733594522422, completionTime=-1 2024-12-07T18:02:02,422 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-07T18:02:02,422 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T18:02:02,425 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T18:02:02,425 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733594582425 2024-12-07T18:02:02,425 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733594642425 2024-12-07T18:02:02,425 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-07T18:02:02,426 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,40469,1733594521129-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:02,426 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,40469,1733594521129-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:02,426 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,40469,1733594521129-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:02,426 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bd53b59592b3:40469, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:02,426 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:02,427 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:02,429 DEBUG [master/bd53b59592b3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T18:02:02,432 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.049sec 2024-12-07T18:02:02,432 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T18:02:02,433 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T18:02:02,433 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T18:02:02,433 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T18:02:02,433 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T18:02:02,433 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,40469,1733594521129-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T18:02:02,433 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,40469,1733594521129-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T18:02:02,434 DEBUG [master/bd53b59592b3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T18:02:02,434 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T18:02:02,434 INFO [master/bd53b59592b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,40469,1733594521129-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:02,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2129cce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:02:02,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd53b59592b3,40469,-1 for getting cluster id 2024-12-07T18:02:02,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:02:02,451 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6c83292e-7a73-4c09-872c-83932733bea9' 2024-12-07T18:02:02,452 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:02:02,452 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6c83292e-7a73-4c09-872c-83932733bea9" 2024-12-07T18:02:02,453 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73503cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:02:02,453 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd53b59592b3,40469,-1] 2024-12-07T18:02:02,453 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:02:02,454 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:02,455 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:02:02,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:02:02,457 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:02:02,459 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd53b59592b3,42669,1733594521275, seqNum=-1] 2024-12-07T18:02:02,460 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:02:02,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:02:02,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bd53b59592b3,40469,1733594521129 2024-12-07T18:02:02,466 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T18:02:02,467 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is bd53b59592b3,40469,1733594521129 2024-12-07T18:02:02,467 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c93f974 2024-12-07T18:02:02,468 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T18:02:02,470 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T18:02:02,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40469 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733594522465', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:02:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40469 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733594522465 2024-12-07T18:02:02,473 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733594522465 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T18:02:02,473 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:02,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40469 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733594522465" procId is: 4 2024-12-07T18:02:02,474 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733594522465 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T18:02:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:02:02,482 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733594522465-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 873778f38c58bc5f6399c6f50e13d119, NAME => 'testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733594522465', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 2024-12-07T18:02:02,490 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733594522465-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:02:02,490 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733594522465-pool-0 {}] regionserver.HRegion(1722): Closing 873778f38c58bc5f6399c6f50e13d119, disabling compactions & flushes 2024-12-07T18:02:02,490 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733594522465-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:02,490 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733594522465-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:02,490 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733594522465-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. after waiting 0 ms 2024-12-07T18:02:02,490 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733594522465-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:02,490 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733594522465-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:02,490 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733594522465-pool-0 {}] regionserver.HRegion(1676): Region close journal for 873778f38c58bc5f6399c6f50e13d119: Waiting for close lock at 1733594522490Disabling compacts and flushes for region at 1733594522490Disabling writes for close at 1733594522490Writing region close event to WAL at 1733594522490Closed at 1733594522490 2024-12-07T18:02:02,491 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733594522465 execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T18:02:02,492 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733594522491"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733594522491"}]},"ts":"1733594522491"} 2024-12-07T18:02:02,495 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T18:02:02,496 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733594522465 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T18:02:02,496 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733594522465","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733594522496"}]},"ts":"1733594522496"} 2024-12-07T18:02:02,498 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733594522465, state=ENABLING in hbase:meta 2024-12-07T18:02:02,498 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {bd53b59592b3=0} racks are {/default-rack=0} 2024-12-07T18:02:02,499 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T18:02:02,499 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T18:02:02,499 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T18:02:02,499 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T18:02:02,499 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T18:02:02,499 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T18:02:02,499 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T18:02:02,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN}] 2024-12-07T18:02:02,500 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN 2024-12-07T18:02:02,501 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN; state=OFFLINE, location=bd53b59592b3,33515,1733594521314; forceNewPlan=false, retain=false 2024-12-07T18:02:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:02:02,652 INFO [bd53b59592b3:40469 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T18:02:02,652 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=873778f38c58bc5f6399c6f50e13d119, regionState=OPENING, regionLocation=bd53b59592b3,33515,1733594521314 2024-12-07T18:02:02,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN because future has completed 2024-12-07T18:02:02,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 873778f38c58bc5f6399c6f50e13d119, server=bd53b59592b3,33515,1733594521314}] 2024-12-07T18:02:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:02:02,810 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T18:02:02,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49491, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T18:02:02,816 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:02,816 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 873778f38c58bc5f6399c6f50e13d119, NAME => 'testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:02:02,816 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733594522465 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,817 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:02:02,817 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,817 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,818 INFO [StoreOpener-873778f38c58bc5f6399c6f50e13d119-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,822 INFO [StoreOpener-873778f38c58bc5f6399c6f50e13d119-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 873778f38c58bc5f6399c6f50e13d119 columnFamilyName family 2024-12-07T18:02:02,822 DEBUG [StoreOpener-873778f38c58bc5f6399c6f50e13d119-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:02,823 INFO [StoreOpener-873778f38c58bc5f6399c6f50e13d119-1 {}] regionserver.HStore(327): Store=873778f38c58bc5f6399c6f50e13d119/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:02:02,823 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,824 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,824 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,824 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,824 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,825 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,831 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:02:02,831 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 873778f38c58bc5f6399c6f50e13d119; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69566305, jitterRate=0.03661872446537018}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T18:02:02,831 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:02,832 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 873778f38c58bc5f6399c6f50e13d119: Running coprocessor pre-open hook at 1733594522817Writing region info on filesystem at 1733594522817Initializing all the Stores at 1733594522817Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594522817Cleaning up temporary data from old regions at 1733594522824 (+7 ms)Running coprocessor post-open hooks at 1733594522832 (+8 ms)Region opened successfully at 1733594522832 2024-12-07T18:02:02,833 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119., pid=6, masterSystemTime=1733594522809 2024-12-07T18:02:02,836 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:02,836 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:02,837 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=873778f38c58bc5f6399c6f50e13d119, regionState=OPEN, openSeqNum=2, regionLocation=bd53b59592b3,33515,1733594521314 2024-12-07T18:02:02,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 873778f38c58bc5f6399c6f50e13d119, server=bd53b59592b3,33515,1733594521314 because future has completed 2024-12-07T18:02:02,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T18:02:02,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 873778f38c58bc5f6399c6f50e13d119, server=bd53b59592b3,33515,1733594521314 in 186 msec 2024-12-07T18:02:02,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T18:02:02,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN in 344 msec 2024-12-07T18:02:02,846 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733594522465 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T18:02:02,846 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733594522465","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733594522846"}]},"ts":"1733594522846"} 2024-12-07T18:02:02,848 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733594522465, state=ENABLED in hbase:meta 2024-12-07T18:02:02,850 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733594522465 execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T18:02:02,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733594522465 in 379 msec 2024-12-07T18:02:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:02:03,102 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733594522465 completed 2024-12-07T18:02:03,103 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733594522465 get assigned. Timeout = 60000ms 2024-12-07T18:02:03,103 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T18:02:03,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733594522465 assigned to meta. Checking AM states. 2024-12-07T18:02:03,108 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T18:02:03,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733594522465 assigned. 2024-12-07T18:02:03,113 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733594522465', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119., hostname=bd53b59592b3,33515,1733594521314, seqNum=2] 2024-12-07T18:02:03,113 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:02:03,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:02:03,122 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 873778f38c58bc5f6399c6f50e13d119 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-07T18:02:03,130 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a1d7c2ff8b374017a07b0fa86bf5456a 2024-12-07T18:02:03,143 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/.tmp/family/a1d7c2ff8b374017a07b0fa86bf5456a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-07T18:02:03,143 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/.tmp/family/a1d7c2ff8b374017a07b0fa86bf5456a 2024-12-07T18:02:03,146 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/.tmp/family/a1d7c2ff8b374017a07b0fa86bf5456a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a 2024-12-07T18:02:03,146 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a, delay=932 ms 2024-12-07T18:02:03,147 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a, entries=2, sequenceid=6, filesize=4.9 K 2024-12-07T18:02:03,148 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 873778f38c58bc5f6399c6f50e13d119 in 26ms, sequenceid=6, compaction requested=false 2024-12-07T18:02:03,148 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733594522465' 2024-12-07T18:02:03,149 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 873778f38c58bc5f6399c6f50e13d119: 2024-12-07T18:02:03,243 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T18:02:04,080 DEBUG [hfile-prefetch-1733594520456 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a1d7c2ff8b374017a07b0fa86bf5456a_0. 2024-12-07T18:02:04,081 DEBUG [hfile-prefetch-1733594520456 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a1d7c2ff8b374017a07b0fa86bf5456a_117. 2024-12-07T18:02:04,082 DEBUG [hfile-prefetch-1733594520456 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a, with total blocks 2, and data blocks 1 2024-12-07T18:02:04,082 DEBUG [hfile-prefetch-1733594520456 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-07T18:02:04,083 DEBUG [hfile-prefetch-1733594520456 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-07T18:02:04,083 DEBUG [hfile-prefetch-1733594520456 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a. Will sleep 100ms and try the verification again. 2024-12-07T18:02:04,149 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;bd53b59592b3:33515,5,FailOnTimeoutGroup] 2024-12-07T18:02:04,149 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bd53b59592b3,33515,1733594521314' ***** 2024-12-07T18:02:04,150 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-07T18:02:04,150 INFO [RS:1;bd53b59592b3:33515 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T18:02:04,163 INFO [RS:1;bd53b59592b3:33515 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.persistence 2024-12-07T18:02:04,163 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T18:02:04,163 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T18:02:04,164 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T18:02:04,164 INFO [RS:1;bd53b59592b3:33515 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T18:02:04,164 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T18:02:04,174 DEBUG [RS:1;bd53b59592b3:33515 {}] bucket.BucketCache(1487): Thread RS:1;bd53b59592b3:33515 started persisting bucket cache to file 2024-12-07T18:02:04,175 DEBUG [RS:1;bd53b59592b3:33515 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T18:02:04,175 DEBUG [RS:1;bd53b59592b3:33515 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T18:02:04,175 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T18:02:04,176 INFO [regionserver/bd53b59592b3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:04,182 DEBUG [RS:1;bd53b59592b3:33515 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T18:02:04,182 DEBUG [RS:1;bd53b59592b3:33515 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733594524175 2024-12-07T18:02:04,182 DEBUG [RS:1;bd53b59592b3:33515 {}] bucket.BucketCache(1509): Thread RS:1;bd53b59592b3:33515 finished persisting bucket cache to file, renaming 2024-12-07T18:02:04,183 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T18:02:04,183 INFO [RS:1;bd53b59592b3:33515 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T18:02:04,183 INFO [RS:1;bd53b59592b3:33515 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T18:02:04,183 DEBUG [hfile-prefetch-1733594520456 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a, with total blocks 2, and data blocks 1 2024-12-07T18:02:04,183 DEBUG [hfile-prefetch-1733594520456 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-07T18:02:04,183 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(3091): Received CLOSE for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,183 DEBUG [hfile-prefetch-1733594520456 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-07T18:02:04,184 INFO [hfile-prefetch-1733594520456 {}] bucket.BucketCache(2331): We found only 0 data blocks cached from a total of 1 for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-12-07T18:02:04,184 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(959): stopping server bd53b59592b3,33515,1733594521314 2024-12-07T18:02:04,184 INFO [RS:1;bd53b59592b3:33515 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T18:02:04,184 INFO [RS:1;bd53b59592b3:33515 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;bd53b59592b3:33515. 2024-12-07T18:02:04,184 DEBUG [hfile-prefetch-1733594520456 {}] hfile.PrefetchExecutor(121): Prefetch completed for a1d7c2ff8b374017a07b0fa86bf5456a 2024-12-07T18:02:04,184 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T18:02:04,185 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 873778f38c58bc5f6399c6f50e13d119, disabling compactions & flushes 2024-12-07T18:02:04,185 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:04,185 DEBUG [RS:1;bd53b59592b3:33515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:02:04,185 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:04,185 DEBUG [RS:1;bd53b59592b3:33515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:04,185 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. after waiting 0 ms 2024-12-07T18:02:04,185 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:04,185 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-07T18:02:04,186 DEBUG [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(1325): Online Regions={873778f38c58bc5f6399c6f50e13d119=testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119.} 2024-12-07T18:02:04,186 DEBUG [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(1351): Waiting on 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,191 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-07T18:02:04,191 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:04,191 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 873778f38c58bc5f6399c6f50e13d119: Waiting for close lock at 1733594524184Running coprocessor pre-close hooks at 1733594524185 (+1 ms)Disabling compacts and flushes for region at 1733594524185Disabling writes for close at 1733594524185Writing region close event to WAL at 1733594524187 (+2 ms)Running coprocessor post-close hooks at 1733594524191 (+4 ms)Closed at 1733594524191 2024-12-07T18:02:04,191 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:04,198 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T18:02:04,198 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T18:02:04,198 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T18:02:04,198 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T18:02:04,201 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T18:02:04,201 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733594524198 2024-12-07T18:02:04,201 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T18:02:04,386 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(976): stopping server bd53b59592b3,33515,1733594521314; all regions closed. 2024-12-07T18:02:04,391 DEBUG [RS:1;bd53b59592b3:33515 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs 2024-12-07T18:02:04,391 INFO [RS:1;bd53b59592b3:33515 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bd53b59592b3%2C33515%2C1733594521314:(num 1733594522006) 2024-12-07T18:02:04,391 DEBUG [RS:1;bd53b59592b3:33515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:04,391 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:04,392 INFO [RS:1;bd53b59592b3:33515 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T18:02:04,392 INFO [RS:1;bd53b59592b3:33515 {}] hbase.ChoreService(370): Chore service for: regionserver/bd53b59592b3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T18:02:04,393 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T18:02:04,393 INFO [regionserver/bd53b59592b3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T18:02:04,393 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T18:02:04,393 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T18:02:04,393 INFO [RS:1;bd53b59592b3:33515 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T18:02:04,393 INFO [RS:1;bd53b59592b3:33515 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33515 2024-12-07T18:02:04,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:02:04,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bd53b59592b3,33515,1733594521314 2024-12-07T18:02:04,423 INFO [RS:1;bd53b59592b3:33515 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T18:02:04,432 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bd53b59592b3,33515,1733594521314] 2024-12-07T18:02:04,440 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bd53b59592b3,33515,1733594521314 already deleted, retry=false 2024-12-07T18:02:04,440 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of bd53b59592b3,33515,1733594521314 on bd53b59592b3,40469,1733594521129 2024-12-07T18:02:04,446 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure bd53b59592b3,33515,1733594521314, splitWal=true, meta=false 2024-12-07T18:02:04,450 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure bd53b59592b3,33515,1733594521314, splitWal=true, meta=false 2024-12-07T18:02:04,450 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for bd53b59592b3,33515,1733594521314 (carryingMeta=false) bd53b59592b3,33515,1733594521314/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@748f2f98[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-07T18:02:04,452 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): bd53b59592b3,33515,1733594521314 had 1 regions 2024-12-07T18:02:04,453 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bd53b59592b3,33515,1733594521314, splitWal=true, meta=false, isMeta: false 2024-12-07T18:02:04,453 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,33515,1733594521314-splitting 2024-12-07T18:02:04,454 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,33515,1733594521314-splitting dir is empty, no logs to split. 2024-12-07T18:02:04,454 INFO [PEWorker-1 {}] master.SplitWALManager(105): bd53b59592b3,33515,1733594521314 WAL count=0, meta=false 2024-12-07T18:02:04,456 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,33515,1733594521314-splitting dir is empty, no logs to split. 2024-12-07T18:02:04,456 INFO [PEWorker-1 {}] master.SplitWALManager(105): bd53b59592b3,33515,1733594521314 WAL count=0, meta=false 2024-12-07T18:02:04,456 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if bd53b59592b3,33515,1733594521314 WAL splitting is done? wals=0, meta=false 2024-12-07T18:02:04,456 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for bd53b59592b3,33515,1733594521314 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,33515,1733594521314-splitting does not exist 2024-12-07T18:02:04,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN}] 2024-12-07T18:02:04,459 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN 2024-12-07T18:02:04,459 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T18:02:04,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:04,532 INFO [RS:1;bd53b59592b3:33515 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T18:02:04,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x100017edcb00002, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:04,532 INFO [RS:1;bd53b59592b3:33515 {}] regionserver.HRegionServer(1031): Exiting; stopping=bd53b59592b3,33515,1733594521314; zookeeper connection closed. 2024-12-07T18:02:04,532 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@503fc459 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@503fc459 2024-12-07T18:02:04,610 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=873778f38c58bc5f6399c6f50e13d119, regionState=OPENING, regionLocation=bd53b59592b3,42669,1733594521275 2024-12-07T18:02:04,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN because future has completed 2024-12-07T18:02:04,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 873778f38c58bc5f6399c6f50e13d119, server=bd53b59592b3,42669,1733594521275}] 2024-12-07T18:02:04,671 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bd53b59592b3:0 server-side Connection retries=45 2024-12-07T18:02:04,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:04,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:04,671 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T18:02:04,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:02:04,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T18:02:04,672 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T18:02:04,672 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T18:02:04,672 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37329 2024-12-07T18:02:04,674 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37329 connecting to ZooKeeper ensemble=127.0.0.1:62026 2024-12-07T18:02:04,674 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:04,674 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:04,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:373290x0, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T18:02:04,699 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37329-0x100017edcb00003 connected 2024-12-07T18:02:04,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-07T18:02:04,699 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-07T18:02:04,700 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T18:02:04,701 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.cache 2024-12-07T18:02:04,701 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T18:02:04,701 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T18:02:04,701 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T18:02:04,702 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T18:02:04,702 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T18:02:04,703 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T18:02:04,703 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-07T18:02:04,703 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-07T18:02:04,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T18:02:04,707 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T18:02:04,711 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-07T18:02:04,712 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T18:02:04,712 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-07T18:02:04,712 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.persistence is complete. 2024-12-07T18:02:04,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37329 2024-12-07T18:02:04,716 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37329 2024-12-07T18:02:04,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37329 2024-12-07T18:02:04,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37329 2024-12-07T18:02:04,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37329 2024-12-07T18:02:04,720 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(746): ClusterId : 6c83292e-7a73-4c09-872c-83932733bea9 2024-12-07T18:02:04,720 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T18:02:04,733 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T18:02:04,733 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T18:02:04,741 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T18:02:04,741 DEBUG [RS:2;bd53b59592b3:37329 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66367620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd53b59592b3/172.17.0.2:0 2024-12-07T18:02:04,751 DEBUG [RS:2;bd53b59592b3:37329 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;bd53b59592b3:37329 2024-12-07T18:02:04,751 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T18:02:04,751 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T18:02:04,751 DEBUG [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T18:02:04,752 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd53b59592b3,40469,1733594521129 with port=37329, startcode=1733594524671 2024-12-07T18:02:04,752 DEBUG [RS:2;bd53b59592b3:37329 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T18:02:04,754 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37141, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T18:02:04,754 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40469 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bd53b59592b3,37329,1733594524671 2024-12-07T18:02:04,754 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40469 {}] master.ServerManager(517): Registering regionserver=bd53b59592b3,37329,1733594524671 2024-12-07T18:02:04,756 DEBUG [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9 2024-12-07T18:02:04,756 DEBUG [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T18:02:04,756 DEBUG [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T18:02:04,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:02:04,766 DEBUG [RS:2;bd53b59592b3:37329 {}] zookeeper.ZKUtil(111): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bd53b59592b3,37329,1733594524671 2024-12-07T18:02:04,766 WARN [RS:2;bd53b59592b3:37329 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T18:02:04,766 INFO [RS:2;bd53b59592b3:37329 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:02:04,766 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bd53b59592b3,37329,1733594524671] 2024-12-07T18:02:04,766 DEBUG [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,37329,1733594524671 2024-12-07T18:02:04,770 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T18:02:04,771 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:04,771 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 873778f38c58bc5f6399c6f50e13d119, NAME => 'testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:02:04,773 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733594522465 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,773 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T18:02:04,773 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:02:04,773 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,773 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,773 INFO [RS:2;bd53b59592b3:37329 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T18:02:04,773 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,773 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T18:02:04,774 INFO [StoreOpener-873778f38c58bc5f6399c6f50e13d119-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,774 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T18:02:04,774 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,774 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,774 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,774 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,774 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,774 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,774 INFO [StoreOpener-873778f38c58bc5f6399c6f50e13d119-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 873778f38c58bc5f6399c6f50e13d119 columnFamilyName family 2024-12-07T18:02:04,774 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bd53b59592b3:0, corePoolSize=2, maxPoolSize=2 2024-12-07T18:02:04,774 DEBUG [StoreOpener-873778f38c58bc5f6399c6f50e13d119-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:02:04,775 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,775 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,775 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,775 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,775 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,775 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bd53b59592b3:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:02:04,775 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:02:04,775 DEBUG [RS:2;bd53b59592b3:37329 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bd53b59592b3:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:02:04,780 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,780 DEBUG [StoreFileOpener-873778f38c58bc5f6399c6f50e13d119-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a, delay=996 ms 2024-12-07T18:02:04,780 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,780 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,780 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,780 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,780 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,37329,1733594524671-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T18:02:04,781 DEBUG [StoreOpener-873778f38c58bc5f6399c6f50e13d119-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a 2024-12-07T18:02:04,781 INFO [StoreOpener-873778f38c58bc5f6399c6f50e13d119-1 {}] regionserver.HStore(327): Store=873778f38c58bc5f6399c6f50e13d119/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:02:04,781 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,782 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,782 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,782 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,782 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,783 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,783 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 873778f38c58bc5f6399c6f50e13d119; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66698389, jitterRate=-0.006116554141044617}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T18:02:04,784 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:04,784 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 873778f38c58bc5f6399c6f50e13d119: Running coprocessor pre-open hook at 1733594524773Writing region info on filesystem at 1733594524773Initializing all the Stores at 1733594524773Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733594524774 (+1 ms)Cleaning up temporary data from old regions at 1733594524782 (+8 ms)Running coprocessor post-open hooks at 1733594524784 (+2 ms)Region opened successfully at 1733594524784 2024-12-07T18:02:04,785 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119., pid=9, masterSystemTime=1733594524769 2024-12-07T18:02:04,787 DEBUG [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:04,787 INFO [RS_OPEN_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:04,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=873778f38c58bc5f6399c6f50e13d119, regionState=OPEN, openSeqNum=10, regionLocation=bd53b59592b3,42669,1733594521275 2024-12-07T18:02:04,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 873778f38c58bc5f6399c6f50e13d119, server=bd53b59592b3,42669,1733594521275 because future has completed 2024-12-07T18:02:04,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-07T18:02:04,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 873778f38c58bc5f6399c6f50e13d119, server=bd53b59592b3,42669,1733594521275 in 176 msec 2024-12-07T18:02:04,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T18:02:04,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733594522465, region=873778f38c58bc5f6399c6f50e13d119, ASSIGN in 335 msec 2024-12-07T18:02:04,795 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server bd53b59592b3,33515,1733594521314 after splitting done 2024-12-07T18:02:04,795 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T18:02:04,795 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd53b59592b3,37329,1733594524671-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,796 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,796 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.Replication(171): bd53b59592b3,37329,1733594524671 started 2024-12-07T18:02:04,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure bd53b59592b3,33515,1733594521314, splitWal=true, meta=false in 354 msec 2024-12-07T18:02:04,809 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:02:04,809 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(1482): Serving as bd53b59592b3,37329,1733594524671, RpcServer on bd53b59592b3/172.17.0.2:37329, sessionid=0x100017edcb00003 2024-12-07T18:02:04,809 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T18:02:04,809 DEBUG [RS:2;bd53b59592b3:37329 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bd53b59592b3,37329,1733594524671 2024-12-07T18:02:04,810 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,37329,1733594524671' 2024-12-07T18:02:04,810 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T18:02:04,810 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T18:02:04,811 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T18:02:04,811 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T18:02:04,811 DEBUG [RS:2;bd53b59592b3:37329 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bd53b59592b3,37329,1733594524671 2024-12-07T18:02:04,811 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd53b59592b3,37329,1733594524671' 2024-12-07T18:02:04,811 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T18:02:04,812 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T18:02:04,812 DEBUG [RS:2;bd53b59592b3:37329 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T18:02:04,812 INFO [RS:2;bd53b59592b3:37329 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T18:02:04,812 INFO [RS:2;bd53b59592b3:37329 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T18:02:04,914 INFO [RS:2;bd53b59592b3:37329 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T18:02:04,915 INFO [RS:2;bd53b59592b3:37329 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bd53b59592b3%2C37329%2C1733594524671, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,37329,1733594524671, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs, maxLogs=55 2024-12-07T18:02:04,934 INFO [RS:2;bd53b59592b3:37329 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/WALs/bd53b59592b3,37329,1733594524671/bd53b59592b3%2C37329%2C1733594524671.1733594524916 2024-12-07T18:02:04,934 DEBUG [RS:2;bd53b59592b3:37329 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T18:02:05,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-07T18:02:05,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-07T18:02:05,036 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T18:02:05,036 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-07T18:02:05,036 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-07T18:02:05,037 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-07T18:02:05,037 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733594522465 2024-12-07T18:02:05,037 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733594522465 Metrics about Tables on a single HBase RegionServer 2024-12-07T18:02:05,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T18:02:05,310 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:02:05,310 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:02:05,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:05,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:05,311 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:02:05,311 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T18:02:05,311 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=223154735, stopped=false 2024-12-07T18:02:05,312 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bd53b59592b3,40469,1733594521129 2024-12-07T18:02:05,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T18:02:05,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T18:02:05,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T18:02:05,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:05,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:05,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:05,374 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T18:02:05,374 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:02:05,375 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:02:05,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:05,375 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:02:05,375 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:02:05,375 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:02:05,375 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bd53b59592b3,42669,1733594521275' ***** 2024-12-07T18:02:05,375 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T18:02:05,375 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bd53b59592b3,37329,1733594524671' ***** 2024-12-07T18:02:05,376 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T18:02:05,376 INFO [RS:0;bd53b59592b3:42669 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T18:02:05,376 INFO [RS:0;bd53b59592b3:42669 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.persistence 2024-12-07T18:02:05,376 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T18:02:05,376 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T18:02:05,376 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T18:02:05,376 INFO [RS:2;bd53b59592b3:37329 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T18:02:05,376 INFO [RS:0;bd53b59592b3:42669 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T18:02:05,376 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T18:02:05,376 INFO [RS:2;bd53b59592b3:37329 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14319508-4508-aeac-b191-3e8f10296d47/bucket.persistence 2024-12-07T18:02:05,377 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T18:02:05,377 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T18:02:05,376 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T18:02:05,376 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T18:02:05,377 INFO [RS:2;bd53b59592b3:37329 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T18:02:05,377 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T18:02:05,380 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T18:02:05,381 INFO [regionserver/bd53b59592b3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:05,384 INFO [regionserver/bd53b59592b3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:05,386 DEBUG [RS:0;bd53b59592b3:42669 {}] bucket.BucketCache(1487): Thread RS:0;bd53b59592b3:42669 started persisting bucket cache to file 2024-12-07T18:02:05,386 DEBUG [RS:0;bd53b59592b3:42669 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T18:02:05,387 DEBUG [RS:0;bd53b59592b3:42669 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T18:02:05,387 DEBUG [RS:2;bd53b59592b3:37329 {}] bucket.BucketCache(1487): Thread RS:2;bd53b59592b3:37329 started persisting bucket cache to file 2024-12-07T18:02:05,387 DEBUG [RS:2;bd53b59592b3:37329 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T18:02:05,387 DEBUG [RS:2;bd53b59592b3:37329 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T18:02:05,389 DEBUG [RS:0;bd53b59592b3:42669 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T18:02:05,389 DEBUG [RS:0;bd53b59592b3:42669 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733594525386 2024-12-07T18:02:05,389 DEBUG [RS:0;bd53b59592b3:42669 {}] bucket.BucketCache(1509): Thread RS:0;bd53b59592b3:42669 finished persisting bucket cache to file, renaming 2024-12-07T18:02:05,390 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T18:02:05,390 INFO [RS:0;bd53b59592b3:42669 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T18:02:05,390 INFO [RS:0;bd53b59592b3:42669 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T18:02:05,390 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(3091): Received CLOSE for 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:05,390 DEBUG [RS:2;bd53b59592b3:37329 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T18:02:05,390 DEBUG [RS:2;bd53b59592b3:37329 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733594525387 2024-12-07T18:02:05,390 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(959): stopping server bd53b59592b3,42669,1733594521275 2024-12-07T18:02:05,390 DEBUG [RS:2;bd53b59592b3:37329 {}] bucket.BucketCache(1509): Thread RS:2;bd53b59592b3:37329 finished persisting bucket cache to file, renaming 2024-12-07T18:02:05,390 INFO [RS:0;bd53b59592b3:42669 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T18:02:05,390 INFO [RS:0;bd53b59592b3:42669 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bd53b59592b3:42669. 2024-12-07T18:02:05,390 DEBUG [RS:0;bd53b59592b3:42669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:02:05,390 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T18:02:05,391 DEBUG [RS:0;bd53b59592b3:42669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:05,391 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T18:02:05,391 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 873778f38c58bc5f6399c6f50e13d119, disabling compactions & flushes 2024-12-07T18:02:05,391 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T18:02:05,391 INFO [RS:2;bd53b59592b3:37329 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T18:02:05,391 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T18:02:05,391 INFO [RS:2;bd53b59592b3:37329 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T18:02:05,391 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:05,391 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T18:02:05,391 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(959): stopping server bd53b59592b3,37329,1733594524671 2024-12-07T18:02:05,391 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T18:02:05,391 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:05,391 INFO [RS:2;bd53b59592b3:37329 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T18:02:05,391 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. after waiting 0 ms 2024-12-07T18:02:05,391 INFO [RS:2;bd53b59592b3:37329 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;bd53b59592b3:37329. 2024-12-07T18:02:05,391 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:05,391 DEBUG [RS:2;bd53b59592b3:37329 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:02:05,391 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-07T18:02:05,391 DEBUG [RS:2;bd53b59592b3:37329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:05,391 DEBUG [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 873778f38c58bc5f6399c6f50e13d119=testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119.} 2024-12-07T18:02:05,391 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(976): stopping server bd53b59592b3,37329,1733594524671; all regions closed. 2024-12-07T18:02:05,391 DEBUG [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 873778f38c58bc5f6399c6f50e13d119 2024-12-07T18:02:05,391 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T18:02:05,391 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T18:02:05,391 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T18:02:05,392 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T18:02:05,392 DEBUG [StoreFileCloser-873778f38c58bc5f6399c6f50e13d119-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/family/a1d7c2ff8b374017a07b0fa86bf5456a 2024-12-07T18:02:05,392 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T18:02:05,392 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T18:02:05,392 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-07T18:02:05,394 DEBUG [RS:2;bd53b59592b3:37329 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs 2024-12-07T18:02:05,395 INFO [RS:2;bd53b59592b3:37329 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bd53b59592b3%2C37329%2C1733594524671:(num 1733594524916) 2024-12-07T18:02:05,395 DEBUG [RS:2;bd53b59592b3:37329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:05,395 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:05,395 INFO [RS:2;bd53b59592b3:37329 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T18:02:05,395 INFO [RS:2;bd53b59592b3:37329 {}] hbase.ChoreService(370): Chore service for: regionserver/bd53b59592b3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T18:02:05,395 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T18:02:05,395 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T18:02:05,395 INFO [regionserver/bd53b59592b3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T18:02:05,395 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T18:02:05,395 INFO [RS:2;bd53b59592b3:37329 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T18:02:05,395 INFO [RS:2;bd53b59592b3:37329 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37329 2024-12-07T18:02:05,397 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/default/testBlockEvictionOnGracefulStop1733594522465/873778f38c58bc5f6399c6f50e13d119/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-07T18:02:05,397 INFO [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:05,397 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 873778f38c58bc5f6399c6f50e13d119: Waiting for close lock at 1733594525390Running coprocessor pre-close hooks at 1733594525391 (+1 ms)Disabling compacts and flushes for region at 1733594525391Disabling writes for close at 1733594525391Writing region close event to WAL at 1733594525392 (+1 ms)Running coprocessor post-close hooks at 1733594525397 (+5 ms)Closed at 1733594525397 2024-12-07T18:02:05,397 DEBUG [RS_CLOSE_REGION-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119. 2024-12-07T18:02:05,403 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bdaaed8621d94e88927a072a71abfd29 2024-12-07T18:02:05,415 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/info/bdaaed8621d94e88927a072a71abfd29 is 209, key is testBlockEvictionOnGracefulStop1733594522465,,1733594522470.873778f38c58bc5f6399c6f50e13d119./info:regioninfo/1733594524787/Put/seqid=0 2024-12-07T18:02:05,416 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/info/bdaaed8621d94e88927a072a71abfd29 2024-12-07T18:02:05,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bd53b59592b3,37329,1733594524671 2024-12-07T18:02:05,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:02:05,417 INFO [RS:2;bd53b59592b3:37329 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T18:02:05,423 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 02c0a14fd5a64dac8543bd11e557ecc5 2024-12-07T18:02:05,431 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bd53b59592b3,37329,1733594524671] 2024-12-07T18:02:05,434 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/ns/02c0a14fd5a64dac8543bd11e557ecc5 is 43, key is default/ns:d/1733594522417/Put/seqid=0 2024-12-07T18:02:05,435 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/ns/02c0a14fd5a64dac8543bd11e557ecc5 2024-12-07T18:02:05,440 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bd53b59592b3,37329,1733594524671 already deleted, retry=false 2024-12-07T18:02:05,440 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bd53b59592b3,37329,1733594524671 expired; onlineServers=1 2024-12-07T18:02:05,442 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aba44dd9abdc40e5b64b17fdac45c436 2024-12-07T18:02:05,453 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/table/aba44dd9abdc40e5b64b17fdac45c436 is 80, key is testBlockEvictionOnGracefulStop1733594522465/table:state/1733594522846/Put/seqid=0 2024-12-07T18:02:05,453 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/table/aba44dd9abdc40e5b64b17fdac45c436 2024-12-07T18:02:05,455 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/info/bdaaed8621d94e88927a072a71abfd29 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/info/bdaaed8621d94e88927a072a71abfd29 2024-12-07T18:02:05,456 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/info/bdaaed8621d94e88927a072a71abfd29, delay=963 ms 2024-12-07T18:02:05,457 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/info/bdaaed8621d94e88927a072a71abfd29, entries=14, sequenceid=13, filesize=7.5 K 2024-12-07T18:02:05,457 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/ns/02c0a14fd5a64dac8543bd11e557ecc5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/ns/02c0a14fd5a64dac8543bd11e557ecc5 2024-12-07T18:02:05,458 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/ns/02c0a14fd5a64dac8543bd11e557ecc5, delay=991 ms 2024-12-07T18:02:05,459 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/ns/02c0a14fd5a64dac8543bd11e557ecc5, entries=2, sequenceid=13, filesize=5.0 K 2024-12-07T18:02:05,460 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/.tmp/table/aba44dd9abdc40e5b64b17fdac45c436 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/table/aba44dd9abdc40e5b64b17fdac45c436 2024-12-07T18:02:05,461 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/table/aba44dd9abdc40e5b64b17fdac45c436, delay=901 ms 2024-12-07T18:02:05,462 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/table/aba44dd9abdc40e5b64b17fdac45c436, entries=2, sequenceid=13, filesize=5.3 K 2024-12-07T18:02:05,462 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=13, compaction requested=false 2024-12-07T18:02:05,463 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/info/bdaaed8621d94e88927a072a71abfd29 2024-12-07T18:02:05,464 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/ns/02c0a14fd5a64dac8543bd11e557ecc5 2024-12-07T18:02:05,464 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/table/aba44dd9abdc40e5b64b17fdac45c436 2024-12-07T18:02:05,468 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-07T18:02:05,468 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T18:02:05,468 INFO [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T18:02:05,468 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733594525391Running coprocessor pre-close hooks at 1733594525391Disabling compacts and flushes for region at 1733594525391Disabling writes for close at 1733594525392 (+1 ms)Obtaining lock to block concurrent updates at 1733594525392Preparing flush snapshotting stores in 1588230740 at 1733594525392Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733594525392Flushing stores of hbase:meta,,1.1588230740 at 1733594525392Flushing 1588230740/info: creating writer at 1733594525392Flushing 1588230740/info: appending metadata at 1733594525415 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733594525415Flushing 1588230740/ns: creating writer at 1733594525418 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733594525434 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733594525434Flushing 1588230740/table: creating writer at 1733594525437 (+3 ms)Flushing 1588230740/table: appending metadata at 1733594525452 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733594525452Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18e23b59: reopening flushed file at 1733594525455 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@391acc33: reopening flushed file at 1733594525457 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e77d8e3: reopening flushed file at 1733594525459 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=13, compaction requested=false at 1733594525462 (+3 ms)Writing region close event to WAL at 1733594525464 (+2 ms)Running coprocessor post-close hooks at 1733594525468 (+4 ms)Closed at 1733594525468 2024-12-07T18:02:05,469 DEBUG [RS_CLOSE_META-regionserver/bd53b59592b3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T18:02:05,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:05,532 INFO [RS:2;bd53b59592b3:37329 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T18:02:05,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37329-0x100017edcb00003, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:05,532 INFO [RS:2;bd53b59592b3:37329 {}] regionserver.HRegionServer(1031): Exiting; stopping=bd53b59592b3,37329,1733594524671; zookeeper connection closed. 2024-12-07T18:02:05,532 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77e9994c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77e9994c 2024-12-07T18:02:05,592 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(976): stopping server bd53b59592b3,42669,1733594521275; all regions closed. 2024-12-07T18:02:05,596 DEBUG [RS:0;bd53b59592b3:42669 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs 2024-12-07T18:02:05,596 INFO [RS:0;bd53b59592b3:42669 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bd53b59592b3%2C42669%2C1733594521275.meta:.meta(num 1733594522350) 2024-12-07T18:02:05,600 DEBUG [RS:0;bd53b59592b3:42669 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/oldWALs 2024-12-07T18:02:05,600 INFO [RS:0;bd53b59592b3:42669 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bd53b59592b3%2C42669%2C1733594521275:(num 1733594522021) 2024-12-07T18:02:05,600 DEBUG [RS:0;bd53b59592b3:42669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:02:05,600 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:02:05,601 INFO [RS:0;bd53b59592b3:42669 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T18:02:05,601 INFO [RS:0;bd53b59592b3:42669 {}] hbase.ChoreService(370): Chore service for: regionserver/bd53b59592b3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-07T18:02:05,602 INFO [RS:0;bd53b59592b3:42669 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T18:02:05,602 INFO [regionserver/bd53b59592b3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T18:02:05,602 INFO [RS:0;bd53b59592b3:42669 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42669 2024-12-07T18:02:05,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bd53b59592b3,42669,1733594521275 2024-12-07T18:02:05,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:02:05,632 INFO [RS:0;bd53b59592b3:42669 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T18:02:05,640 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bd53b59592b3,42669,1733594521275] 2024-12-07T18:02:05,648 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bd53b59592b3,42669,1733594521275 already deleted, retry=false 2024-12-07T18:02:05,648 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bd53b59592b3,42669,1733594521275 expired; onlineServers=0 2024-12-07T18:02:05,648 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bd53b59592b3,40469,1733594521129' ***** 2024-12-07T18:02:05,648 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T18:02:05,649 INFO [M:0;bd53b59592b3:40469 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T18:02:05,649 INFO [M:0;bd53b59592b3:40469 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T18:02:05,649 DEBUG [M:0;bd53b59592b3:40469 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T18:02:05,649 DEBUG [M:0;bd53b59592b3:40469 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T18:02:05,649 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T18:02:05,649 DEBUG [master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.large.0-1733594521691 {}] cleaner.HFileCleaner(306): Exit Thread[master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.large.0-1733594521691,5,FailOnTimeoutGroup] 2024-12-07T18:02:05,649 DEBUG [master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.small.0-1733594521695 {}] cleaner.HFileCleaner(306): Exit Thread[master/bd53b59592b3:0:becomeActiveMaster-HFileCleaner.small.0-1733594521695,5,FailOnTimeoutGroup] 2024-12-07T18:02:05,649 INFO [M:0;bd53b59592b3:40469 {}] hbase.ChoreService(370): Chore service for: master/bd53b59592b3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T18:02:05,649 INFO [M:0;bd53b59592b3:40469 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T18:02:05,650 DEBUG [M:0;bd53b59592b3:40469 {}] master.HMaster(1795): Stopping service threads 2024-12-07T18:02:05,650 INFO [M:0;bd53b59592b3:40469 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T18:02:05,650 INFO [M:0;bd53b59592b3:40469 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T18:02:05,650 INFO [M:0;bd53b59592b3:40469 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T18:02:05,650 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T18:02:05,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T18:02:05,657 DEBUG [M:0;bd53b59592b3:40469 {}] zookeeper.ZKUtil(347): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T18:02:05,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:02:05,657 WARN [M:0;bd53b59592b3:40469 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T18:02:05,657 INFO [M:0;bd53b59592b3:40469 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/.lastflushedseqids 2024-12-07T18:02:05,662 INFO [M:0;bd53b59592b3:40469 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T18:02:05,662 INFO [M:0;bd53b59592b3:40469 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T18:02:05,663 DEBUG [M:0;bd53b59592b3:40469 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T18:02:05,663 INFO [M:0;bd53b59592b3:40469 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:05,663 DEBUG [M:0;bd53b59592b3:40469 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:05,663 DEBUG [M:0;bd53b59592b3:40469 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T18:02:05,663 DEBUG [M:0;bd53b59592b3:40469 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:05,663 INFO [M:0;bd53b59592b3:40469 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-12-07T18:02:05,671 DEBUG [M:0;bd53b59592b3:40469 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3ecd0053413a4cc3837c981d14b57a13 2024-12-07T18:02:05,683 DEBUG [M:0;bd53b59592b3:40469 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3ecd0053413a4cc3837c981d14b57a13 is 82, key is hbase:meta,,1/info:regioninfo/1733594522378/Put/seqid=0 2024-12-07T18:02:05,684 INFO [M:0;bd53b59592b3:40469 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3ecd0053413a4cc3837c981d14b57a13 2024-12-07T18:02:05,690 DEBUG [M:0;bd53b59592b3:40469 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a8d900a013b74f2381704dc867b596a8 2024-12-07T18:02:05,704 DEBUG [M:0;bd53b59592b3:40469 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a8d900a013b74f2381704dc867b596a8 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733594522851/Put/seqid=0 2024-12-07T18:02:05,704 INFO [M:0;bd53b59592b3:40469 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a8d900a013b74f2381704dc867b596a8 2024-12-07T18:02:05,711 DEBUG [M:0;bd53b59592b3:40469 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ef73c0cd77614c3eac18190851c6b7ea 2024-12-07T18:02:05,723 DEBUG [M:0;bd53b59592b3:40469 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ef73c0cd77614c3eac18190851c6b7ea is 69, key is bd53b59592b3,37329,1733594524671/rs:state/1733594524755/Put/seqid=0 2024-12-07T18:02:05,723 INFO [M:0;bd53b59592b3:40469 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ef73c0cd77614c3eac18190851c6b7ea 2024-12-07T18:02:05,725 INFO [M:0;bd53b59592b3:40469 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ef73c0cd77614c3eac18190851c6b7ea 2024-12-07T18:02:05,725 DEBUG [M:0;bd53b59592b3:40469 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3ecd0053413a4cc3837c981d14b57a13 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3ecd0053413a4cc3837c981d14b57a13 2024-12-07T18:02:05,727 INFO [M:0;bd53b59592b3:40469 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3ecd0053413a4cc3837c981d14b57a13, entries=8, sequenceid=83, filesize=5.5 K 2024-12-07T18:02:05,727 DEBUG [M:0;bd53b59592b3:40469 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a8d900a013b74f2381704dc867b596a8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a8d900a013b74f2381704dc867b596a8 2024-12-07T18:02:05,729 INFO [M:0;bd53b59592b3:40469 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a8d900a013b74f2381704dc867b596a8, entries=9, sequenceid=83, filesize=6.4 K 2024-12-07T18:02:05,729 DEBUG [M:0;bd53b59592b3:40469 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ef73c0cd77614c3eac18190851c6b7ea as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ef73c0cd77614c3eac18190851c6b7ea 2024-12-07T18:02:05,731 INFO [M:0;bd53b59592b3:40469 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ef73c0cd77614c3eac18190851c6b7ea 2024-12-07T18:02:05,731 INFO [M:0;bd53b59592b3:40469 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/869a82bb-82d2-5f64-f787-001c88359eda/fefa94ac-cf17-18d5-3fe4-ded6ab5d44c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ef73c0cd77614c3eac18190851c6b7ea, entries=3, sequenceid=83, filesize=5.3 K 2024-12-07T18:02:05,732 INFO [M:0;bd53b59592b3:40469 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=83, compaction requested=false 2024-12-07T18:02:05,733 INFO [M:0;bd53b59592b3:40469 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:02:05,733 DEBUG [M:0;bd53b59592b3:40469 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733594525662Disabling compacts and flushes for region at 1733594525662Disabling writes for close at 1733594525663 (+1 ms)Obtaining lock to block concurrent updates at 1733594525663Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733594525663Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1733594525663Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733594525664 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733594525664Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733594525683 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733594525683Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733594525685 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733594525703 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733594525703Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733594525706 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733594525722 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733594525722Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d4c9325: reopening flushed file at 1733594525725 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12d75e7b: reopening flushed file at 1733594525727 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@469f4012: reopening flushed file at 1733594525729 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=83, compaction requested=false at 1733594525732 (+3 ms)Writing region close event to WAL at 1733594525733 (+1 ms)Closed at 1733594525733 2024-12-07T18:02:05,734 INFO [M:0;bd53b59592b3:40469 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T18:02:05,734 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T18:02:05,734 INFO [M:0;bd53b59592b3:40469 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40469 2024-12-07T18:02:05,735 INFO [M:0;bd53b59592b3:40469 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T18:02:05,740 INFO [RS:0;bd53b59592b3:42669 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T18:02:05,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:05,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42669-0x100017edcb00001, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:05,740 INFO [RS:0;bd53b59592b3:42669 {}] regionserver.HRegionServer(1031): Exiting; stopping=bd53b59592b3,42669,1733594521275; zookeeper connection closed. 2024-12-07T18:02:05,741 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@478c7c89 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@478c7c89 2024-12-07T18:02:05,741 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-07T18:02:05,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:05,848 INFO [M:0;bd53b59592b3:40469 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T18:02:05,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40469-0x100017edcb00000, quorum=127.0.0.1:62026, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:02:05,851 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T18:02:05,853 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-07T18:02:05,853 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:02:05,863 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=326 (was 354), ProcessCount=11 (was 11), AvailableMemoryMB=20376 (was 20452)